Added CI workflow to publish CVAT images (cvat-ai#2766)

* Configured CI workflow for public images * updated license headers * align list item indent rule for remarklint and prettier * updated changelog Co-authored-by: Nikita Manovich <[email protected]>
kenu · Feb 23, 2021 · 5b99e99 · 5b99e99
1 parent a262d24
commit 5b99e99
Show file tree

Hide file tree

Showing 13 changed files with 168 additions and 69 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -23,16 +23,16 @@ jobs:
           DJANGO_SU_EMAIL: '[email protected]'
           DJANGO_SU_PASSWORD: '12qwaszx'
         run: |
-          docker-compose -f docker-compose.yml -f docker-compose.ci.yml build
-          docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
-          docker-compose -f docker-compose.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test && mv ./reports/coverage/lcov.info ${CONTAINER_COVERAGE_DATA_DIR} && chmod a+rwx ${CONTAINER_COVERAGE_DATA_DIR}/lcov.info'
           docker-compose up -d
           docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
       - name: Code instrumentation
         run: |
           npm ci
           npm run coverage
-          docker-compose up -d --build
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml up -d --build
       - name: End-to-end testing
         run: |
           cd ./tests

diff --git a/.github/workflows/publish_docker_images.yml b/.github/workflows/publish_docker_images.yml
@@ -0,0 +1,54 @@
+name: Publish Docker images
+on:
+  release:
+    types: [published]
+
+jobs:
+  build_and_push_to_registry:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+            node-version: 12
+
+      - name: Build images
+        run: |
+          CLAM_AV=yes INSTALL_SOURCES=yes docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml build
+
+      - name: Run unit tests
+        env:
+          HOST_COVERAGE_DATA_DIR: ${{ github.workspace }}
+          CONTAINER_COVERAGE_DATA_DIR: '/coverage_data'
+          DJANGO_SU_NAME: 'admin'
+          DJANGO_SU_EMAIL: '[email protected]'
+          DJANGO_SU_PASSWORD: '12qwaszx'
+        run: |
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'coverage run -a manage.py test cvat/apps utils/cli'
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash -c 'cd cvat-data && npm ci && cd ../cvat-core && npm ci && npm run test'
+          docker-compose up -d
+          docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
+
+      - name: Run end-to-end tests
+        run: |
+          cd ./tests
+          npm ci
+          npx cypress run --headless --browser chrome
+      - name: Login to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Push to Docker Hub
+        env:
+          DOCKERHUB_WORKSPACE: 'openvino'
+          SERVER_IMAGE_REPO: 'cvat_server'
+          UI_IMAGE_REPO: 'cvat_ui'
+        run: |
+          docker tag "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${SERVER_IMAGE_REPO}:latest"
+
+          docker tag "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest" "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:${{ github.event.release.tag_name }}"
+          docker push "${DOCKERHUB_WORKSPACE}/${UI_IMAGE_REPO}:latest"
diff --git a/.github/workflows/schedule.yml b/.github/workflows/schedule.yml
@@ -18,7 +18,7 @@ jobs:
           DJANGO_SU_PASSWORD: "12qwaszx"
           API_ABOUT_PAGE: "localhost:8080/api/v1/server/about"
         run: |
-          docker-compose -f docker-compose.yml -f ./tests/docker-compose.email.yml up -d --build
+          docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f ./tests/docker-compose.email.yml up -d --build
           /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
           docker exec -i cvat /bin/bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('${DJANGO_SU_NAME}', '${DJANGO_SU_EMAIL}', '${DJANGO_SU_PASSWORD}')\" | python3 ~/manage.py shell"
       - name: End-to-end testing

diff --git a/.remarkrc.js b/.remarkrc.js
@@ -3,7 +3,7 @@ exports.settings = { bullet: '*', paddedTable: false };
 exports.plugins = [
     'remark-preset-lint-recommended',
     'remark-preset-lint-consistent',
-    ['remark-preset-lint-markdown-style-guide', 'mixed'],
+    ['remark-lint-list-item-indent', 'space'],
     ['remark-lint-no-dead-urls', { skipOffline: true }],
     ['remark-lint-maximum-line-length', 120],
     ['remark-lint-maximum-heading-length', 120],

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Intelligent scissors with OpenCV javascript (<https://github.com/openvinotoolkit/cvat/pull/2689>)
 - [Inside Outside Guidence](https://github.com/shiyinzhang/Inside-Outside-Guidance) serverless
   function for interative segmentation
+- Pre-built [cvat_server](https://hub.docker.com/r/openvino/cvat_server) and
+  [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui) images were published on DockerHub (<https://github.com/openvinotoolkit/cvat/pull/2766>)
 
 ### Changed
 

diff --git a/Dockerfile b/Dockerfile
@@ -34,10 +34,12 @@ RUN curl -sL https://github.com/cisco/openh264/archive/v${OPENH264_VERSION}.tar.
     make -j5 && make install PREFIX=${PREFIX} && make clean
 
 WORKDIR /tmp/ffmpeg
-RUN curl -sLO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
-    tar -jx --strip-components=1 -f ffmpeg-${FFMPEG_VERSION}.tar.bz2 && \
+RUN curl -sL https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.bz2 --output - | \
+    tar -jx --strip-components=1 && \
     ./configure --disable-nonfree --disable-gpl --enable-libopenh264 --enable-shared --disable-static --prefix="${PREFIX}" && \
-    make -j5 && make install && make distclean
+    # make clean keeps the configuration files that let to know how the original sources were used to create the binary
+    make -j5 && make install && make clean && \
+    tar -zcf "/tmp/ffmpeg-$FFMPEG_VERSION.tar.gz" . && mv "/tmp/ffmpeg-$FFMPEG_VERSION.tar.gz" .
 
 # Install requirements
 RUN python3 -m venv /opt/venv
@@ -93,8 +95,17 @@ RUN apt-get update && \
     rm -rf /var/lib/apt/lists/* && \
     echo 'application/wasm wasm' >> /etc/mime.types
 
-ARG CLAM_AV
-ENV CLAM_AV=${CLAM_AV}
+# Add a non-root user
+ENV USER=${USER}
+ENV HOME /home/${USER}
+RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \
+    if [ -z ${socks_proxy} ]; then \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
+    else \
+        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
+    fi
+
+ARG CLAM_AV="no"
 RUN if [ "$CLAM_AV" = "yes" ]; then \
         apt-get update && \
         apt-get --no-install-recommends install -yq \
@@ -106,16 +117,6 @@ RUN if [ "$CLAM_AV" = "yes" ]; then \
         rm -rf /var/lib/apt/lists/*; \
     fi
 
-# Add a non-root user
-ENV USER=${USER}
-ENV HOME /home/${USER}
-RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \
-    if [ -z ${socks_proxy} ]; then \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \
-    else \
-        echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \
-    fi
-
 ARG INSTALL_SOURCES='no'
 WORKDIR ${HOME}/sources
 RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \
@@ -131,7 +132,7 @@ RUN if [ "$INSTALL_SOURCES" = "yes" ]; then \
             done &&                                           \
         rm -rf /var/lib/apt/lists/*;                          \
     fi
-COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.bz2 ${HOME}/sources/
+COPY --from=build-image /tmp/openh264/openh264*.tar.gz /tmp/ffmpeg/ffmpeg*.tar.gz ${HOME}/sources/
 
 # Copy python virtual enviroment and FFmpeg binaries from build-image
 COPY --from=build-image /opt/venv /opt/venv

diff --git a/Dockerfile.ci b/Dockerfile.ci
@@ -1,4 +1,4 @@
-FROM cvat/server
+FROM openvino/cvat_server
 
 ENV DJANGO_CONFIGURATION=testing
 USER root

diff --git a/LICENSE b/LICENSE
@@ -20,3 +20,12 @@ OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE
 OR OTHER DEALINGS IN THE SOFTWARE.
 
+This software uses LGPL licensed libraries from the [FFmpeg](https://www.ffmpeg.org) project.
+The exact steps on how FFmpeg was configured and compiled can be found in the [Dockerfile](Dockerfile).
+
+FFmpeg is an open source framework licensed under LGPL and GPL.
+See https://www.ffmpeg.org/legal.html. You are solely responsible
+for determining if your use of FFmpeg requires any
+additional licenses. Intel is not responsible for obtaining any
+such licenses, nor liable for any licensing fees due in
+connection with your use of FFmpeg.
diff --git a/README.md b/README.md
@@ -99,6 +99,13 @@ Limitations:
 - No more than 10 tasks per user
 - Uploaded data is limited to 500Mb
 
+## Prebuilt Docker images
+
+Prebuilt docker images for CVAT releases are available on Docker Hub:
+
+- [cvat_server](https://hub.docker.com/r/openvino/cvat_server)
+- [cvat_ui](https://hub.docker.com/r/openvino/cvat_ui)
+
 ## REST API
 
 Automatically generated Swagger documentation for Django REST API is available
@@ -112,6 +119,16 @@ name. Example - `ALLOWED_HOSTS: 'localhost, 127.0.0.1'`.
 
 Code released under the [MIT License](https://opensource.org/licenses/MIT).
 
+This software uses LGPL licensed libraries from the [FFmpeg](https://www.ffmpeg.org) project.
+The exact steps on how FFmpeg was configured and compiled can be found in the [Dockerfile](Dockerfile).
+
+FFmpeg is an open source framework licensed under LGPL and GPL.
+See [https://www.ffmpeg.org/legal.html](https://www.ffmpeg.org/legal.html). You are solely responsible
+for determining if your use of FFmpeg requires any
+additional licenses. Intel is not responsible for obtaining any
+such licenses, nor liable for any licensing fees due in
+connection with your use of FFmpeg.
+
 ## Questions
 
 CVAT usage related questions or unclear concepts can be posted in our

diff --git a/cvat/apps/documentation/installation.md b/cvat/apps/documentation/installation.md
@@ -81,18 +81,18 @@ server. Proxy is an advanced topic and it is not covered by the guide.
   cd cvat
   ```
 
-- Build docker images by default. It will take some time to download public
-  docker image ubuntu:16.04 and install all necessary ubuntu packages to run
-  CVAT server.
+- Run docker containers. It will take some time to download the latest CVAT
+  release and other required images like postgres, redis, etc. from DockerHub and create containers.
 
-  ```bash
-  docker-compose build
+  ```sh
+  docker-compose up -d
   ```
 
-- Run docker containers. It will take some time to download public docker
-  images like postgres:10.3-alpine, redis:4.0.5-alpine and create containers.
+- Alternative: if you want to build the images locally with unreleased changes
+  run the following command. It will take some time to build CVAT images.
 
-  ```sh
+  ```bash
+  docker-compose -f docker-compose.yml -f docker-compose.dev.yml build
   docker-compose up -d
   ```
 
@@ -149,23 +149,23 @@ server. Proxy is an advanced topic and it is not covered by the guide.
 - Clone _CVAT_ source code from the
   [GitHub repository](https://github.com/opencv/cvat).
 
-  ```bash
+  ```sh
   git clone https://github.com/opencv/cvat
   cd cvat
   ```
 
-- Build docker images by default. It will take some time to download public
-  docker image ubuntu:16.04 and install all necessary ubuntu packages to run
-  CVAT server.
+- Run docker containers. It will take some time to download the latest CVAT
+  release and other required images like postgres, redis, etc. from DockerHub and create containers.
 
-  ```bash
-  docker-compose build
+  ```sh
+  docker-compose up -d
   ```
 
-- Run docker containers. It will take some time to download public docker
-  images like postgres:10.3-alpine, redis:4.0.5-alpine and create containers.
+- Alternative: if you want to build the images locally with unreleased changes
+  run the following command. It will take some time to build CVAT images.
 
   ```sh
+  docker-compose -f docker-compose.yml -f docker-compose.dev.yml build
   docker-compose up -d
   ```
 
@@ -221,18 +221,18 @@ server. Proxy is an advanced topic and it is not covered by the guide.
   cd cvat
   ```
 
-- Build docker images by default. It will take some time to download public
-  docker image ubuntu:16.04 and install all necessary ubuntu packages to run
-  CVAT server.
+- Run docker containers. It will take some time to download the latest CVAT
+  release and other required images like postgres, redis, etc. from DockerHub and create containers.
 
-  ```bash
-  docker-compose build
+  ```sh
+  docker-compose up -d
   ```
 
-- Run docker containers. It will take some time to download public docker
-  images like postgres:10.3-alpine, redis:4.0.5-alpine and create containers.
+- Alternative: if you want to build the images locally with unreleased changes
+  run the following command. It will take some time to build CVAT images.
 
   ```sh
+  docker-compose -f docker-compose.yml -f docker-compose.dev.yml build
   docker-compose up -d
   ```
 

diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml
@@ -1,3 +1,8 @@
+#
+# Copyright (C) 2018-2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
 version: '3.3'
 
 services:

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -0,0 +1,31 @@
+#
+# Copyright (C) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+#
+version: '3.3'
+
+services:
+  cvat:
+    build:
+      context: .
+      args:
+        http_proxy:
+        https_proxy:
+        no_proxy: nuclio,${no_proxy}
+        socks_proxy:
+        USER: 'django'
+        DJANGO_CONFIGURATION: 'production'
+        TZ: 'Etc/UTC'
+        CLAM_AV:
+        INSTALL_SOURCES:
+
+  cvat_ui:
+    build:
+      context: .
+      args:
+        http_proxy:
+        https_proxy:
+        no_proxy:
+        socks_proxy:
+      dockerfile: Dockerfile.ui
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,5 +1,5 @@
 #
-# Copyright (C) 2018-2020 Intel Corporation
+# Copyright (C) 2018-2021 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 #
@@ -32,22 +32,11 @@ services:
 
   cvat:
     container_name: cvat
-    image: cvat/server
+    image: openvino/cvat_server
     restart: always
     depends_on:
       - cvat_redis
       - cvat_db
-    build:
-      context: .
-      args:
-        http_proxy:
-        https_proxy:
-        no_proxy: nuclio,${no_proxy}
-        socks_proxy:
-        USER: 'django'
-        DJANGO_CONFIGURATION: 'production'
-        TZ: 'Etc/UTC'
-        CLAM_AV: 'no'
     environment:
       DJANGO_MODWSGI_EXTRA_ARGS: ''
       ALLOWED_HOSTS: '*'
@@ -61,17 +50,8 @@ services:
 
   cvat_ui:
     container_name: cvat_ui
-    image: cvat/ui
+    image: openvino/cvat_ui
     restart: always
-    build:
-      context: .
-      args:
-        http_proxy:
-        https_proxy:
-        no_proxy:
-        socks_proxy:
-      dockerfile: Dockerfile.ui
-
     networks:
       default:
         aliases: