Simplify docker-compose workflow (#187, #188)

.mk/docker.mk

@@ -16,7 +16,7 @@ docker-setup-update:
 
 ## Run application using docker-compose
 docker-run: docker-setup
-	@scripts/docker-run.sh
+	sudo docker-compose up -d
 
 
 .PHONY: docker-stop
@@ -29,20 +29,14 @@ docker-stop:
 
 ## Build docker images for docker-compose application
 docker-build:
-	sudo docker-compose build
+	@scripts/docker-build.sh
 
-.PHONY: docker-rebuild
 
-## Rebuild docker images
-docker-rebuild:
-	sudo docker-compose rm -s -f
-	sudo docker-compose build
-
-.PHONY: docker-update
+.PHONY: docker-pull
 
 ## Update docker images (rebuild local or pull latest from repository depending on configuration).
-docker-update:
-	@scripts/docker-update.sh
+docker-pull:
+	sudo docker-compose pull
 
 .PHONY: docker-purge
 

Makefile

@@ -26,20 +26,20 @@ setup: docker-setup
 ## Update docker-compose application (regenerate .env file)
 setup-update: docker-setup-update
 
-.PHONY: rebuild
-
-## Rebuild docker-compose images
-rebuild: docker-rebuild
-
-.PHONY: update
-
-## Update images of the docker-compose application
-update: docker-update
-
 .PHONY: purge
 
 ## Remove docker-compose application and all its images and volumes.
 purge: docker-purge
 
 # Define default goal
 .DEFAULT_GOAL := help
+
+.PHONY: build
+
+## Build Docker images locally.
+build: docker-build
+
+.PHONY: pull
+
+## Pull images from Docker Hub
+pull: docker-pull

config.yaml

@@ -24,7 +24,7 @@ processing:
 
 database:
   use: true
-  uri: postgres://postgres:admin@postgres:5432/videodeduplicationdb
+  uri: postgres://postgres:admin@localhost:5432/videodeduplicationdb
 
 templates:
   source_path: data/templates/test-group/CCSI Object Recognition External/

docker-compose.yml

@@ -31,7 +31,13 @@ services:
       - postgres-compose-network
 
   dedup-app:
-    runtime: nvidia
+    image: "johnhbenetech/videodeduplication:${BENETECH_RUNTIME:-gpu}${BENETECH_MODE}"
+    build:
+      context: .
+      dockerfile: "docker/Dockerfile.dedup-${BENETECH_RUNTIME:-gpu}"
+      args:
+        GIT_HASH: "${GIT_HASH}"
+    runtime: "${BENETECH_DOCKER_RUNTIME:-nvidia}"
     command: bash -ic "bash serve_jupyter.sh"
     environment:
       PGADMIN_DEFAULT_EMAIL: "[email protected]"
@@ -46,22 +52,18 @@ services:
       - postgres
     networks:
       - postgres-compose-network
-
   server:
-    # This could be replaced by the online repository reference
-    # if you don't want to built a local docker image
+    image: "johnhbenetech/videodeduplication:server${BENETECH_MODE}"
     build:
       context: .
       dockerfile: docker/Dockerfile.server
+      args:
+        GIT_HASH: "${GIT_HASH}"
     environment:
       SERVER_HOST: "0.0.0.0"
       SERVER_PORT: "5000"
       STATIC_FOLDER: "static"
-      DATABASE_HOST: "postgres"
-      DATABASE_PORT: "5432"
-      DATABASE_NAME: "videodeduplicationdb"
-      DATABASE_USER: "postgres"
-      DATABASE_PASS: "admin"
+      DATABASE_URI: "postgres://postgres:admin@postgres:5432/videodeduplicationdb"
       VIDEO_FOLDER: "/project/data/"
       THUMBNAIL_CACHE_FOLDER: "/project/thumbnails"
       THUMBNAIL_CACHE_CAP: 10000

docker/Dockerfile.dedup-cpu

@@ -31,3 +31,6 @@ RUN apt update
 RUN apt-get install -y libsm6 libxext6 libxrender-dev nano libgl1-mesa-glx mediainfo
 
 RUN /bin/bash -c "source activate winnow &&  bash scripts/run-tests.sh"
+
+ARG GIT_HASH=unspecified
+LABEL git_hash=$GIT_HASH

docker/Dockerfile.dedup-gpu

@@ -29,3 +29,6 @@ RUN /bin/bash -c "source activate winnow && \
 RUN apt update 
 
 RUN apt-get install -y libsm6 libxext6 libxrender-dev nano libgl1-mesa-glx mediainfo
+
+ARG GIT_HASH=unspecified
+LABEL git_hash=$GIT_HASH

docker/Dockerfile.server

@@ -28,4 +28,6 @@ RUN pip install -r requirements.txt
 
 COPY --from=web /web/build /server/static
 
+ARG GIT_HASH=unspecified
+LABEL git_hash=$GIT_HASH
 CMD python -m server.main

scripts/docker-build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# This script builds Docker images from the local
+# repository and tag them as development version.
+
+export BENETECH_MODE=""
+
+if [ -f ".env" ]; then
+  source .env
+fi
+
+if [ "$BENETECH_MODE" = "-dev" ]; then
+  set -x
+  sudo docker-compose rm -s -f
+  sudo docker-compose build --build-arg GIT_HASH="$(git rev-parse --short HEAD)" --parallel --no-cache
+else
+  set -x
+  sudo docker build --build-arg GIT_HASH="$(git rev-parse --short HEAD)" --no-cache -t "johnhbenetech/videodeduplication:${BENETECH_RUNTIME:-gpu}-dev" . -f "docker/Dockerfile.dedup-${BENETECH_RUNTIME:-gpu}"
+  sudo docker build --build-arg GIT_HASH="$(git rev-parse --short HEAD)" --no-cache -t "johnhbenetech/videodeduplication:server-dev" . -f "docker/Dockerfile.server"
+fi

scripts/docker-setup.sh

@@ -43,39 +43,40 @@ fi
 # Read source data location
 if [ "$FORCE_UPDATE" = "YES" ] || ! [ -d "$BENETECH_DATA_LOCATION" ]; then
   DIRTY=yes
-  read-dir-path BENETECH_DATA_LOCATION "Please specify the root folder with your video files (use Tab for auto-complete)"
+  tput setaf 6; echo "Please specify the root folder with your video files (use Tab for auto-complete)."; tput sgr0;
+  read-dir-path BENETECH_DATA_LOCATION "Data folder path"
   echo
 fi
 
 # Choose data analysis runtime
 if [ "$FORCE_UPDATE" = "YES" ] || [ -z "$BENETECH_RUNTIME" ]; then
   DIRTY=yes
   tput setaf 6; echo "Would you like to use GPU for data processing?"; tput sgr0;
-  choose BENETECH_RUNTIME GPU="Use GPU for data processing." CPU="Use CPU for data processing."
+  choose BENETECH_RUNTIME gpu="Use GPU for data processing." cpu="Use CPU for data processing."
+  if [ "$BENETECH_RUNTIME" = "nvidia" ]; then
+    BENETECH_DOCKER_RUNTIME="nvidia"
+  else
+    BENETECH_DOCKER_RUNTIME="runc"
+  fi
   echo
 fi
 
+
 # Decide whether to use prebuilt images
-if [ "$FORCE_UPDATE" = "YES" ] || [ -z "$BENETECH_PREBUILT" ]; then
+if [ "$FORCE_UPDATE" = "YES" ] || [ -z "${BENETECH_MODE+x}" ]; then
   DIRTY=yes
-  tput setaf 6; echo "Would you like to use pre-built Docker images?"; tput sgr0;
-  choose BENETECH_PREBUILT YES="Pull pre-built images from Docker Hub." NO="Build images locally."
+  tput setaf 6; echo "Would you like to use production Docker images?"; tput sgr0;
+  choose BENETECH_MODE ''="Use production images." '-dev'="Pull the latest dev-images or build images locally."
   echo
-
-  # Ask if user would like to use dev or prod images
-  if [ "$BENETECH_PREBUILT" = "YES" ]; then
-    tput setaf 6; echo "Would you like to use production Docker images?"; tput sgr0;
-    choose BENETECH_MODE ''="Use production images." '-dev'="Use dev-images."
-    echo
-  fi
 fi
 
 # Write data to the .env file
 if [ -n "$DIRTY" ]; then
   {
+    echo "# This file is generated by scripts/docker-setup.sh"
     echo "BENETECH_DATA_LOCATION=$BENETECH_DATA_LOCATION"
     echo "BENETECH_RUNTIME=$BENETECH_RUNTIME"
-    echo "BENETECH_PREBUILT=$BENETECH_PREBUILT"
+    echo "BENETECH_DOCKER_RUNTIME=$BENETECH_DOCKER_RUNTIME"
     echo "BENETECH_MODE=$BENETECH_MODE"
   } > .env
   tput setaf 2; echo -n "OK"; tput sgr0;

scripts/docker-update.sh

@@ -20,7 +20,7 @@ fi
 if [ "$BENETECH_RUNTIME" = "GPU" ] && [ "$BENETECH_PREBUILT" = "NO" ]; then
   set -x
   sudo docker-compose rm -s -f
-	sudo docker-compose build
+	sudo docker-compose build --parallel --no-cache
 elif [ "$BENETECH_RUNTIME" = "CPU" ] && [ "$BENETECH_PREBUILT" = "NO" ]; then
   set -x
   sudo docker-compose rm -s -f