chore: fix up naming and variables

SwissDataScienceCenter · Mar 1, 2023 · f7fa4c6 · f7fa4c6
1 parent 50e5c68
commit f7fa4c6
Show file tree

Hide file tree

Showing 7 changed files with 69 additions and 120 deletions.
diff --git a/.github/workflows/build-and-push-to-docker.yml b/.github/workflows/build-and-push-to-docker.yml
@@ -91,10 +91,10 @@ jobs:
         # needed by the makefile - these are generated dynamically
         # and hence they are not part of the env setup above
         export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }}
-        export DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}"
+        export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}"
         export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
         make py 
-        echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${DOCKER_LABEL}" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT
 
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -172,15 +172,14 @@ jobs:
         docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
 
         # these are set dynamically so not part of env
-        export CUDA_BASE_IMAGE="renku/renkulab-py:$DEFAULT_PYTHON_VERSION-$LABEL" 
         export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}" 
         export CUDA_CUDART_PACKAGE="${{ matrix.CUDA_CUDART_PACKAGE }}" 
         export CUDA_VERSION="${{ matrix.CUDA_VERSION }}" 
-        export DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}"
+        export CUDA_DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}"
         export EXTRA_LIBRARIES="${{ matrix.EXTRA_LIBRARIES }}" 
         export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}" 
         make cuda
-        echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$DOCKER_LABEL" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT
         
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -210,6 +209,7 @@ jobs:
       matrix:
         EXTENSIONS:
           - vnc
+          - batch
 
     steps:
     - name: Docker Login
@@ -231,10 +231,8 @@ jobs:
         docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
         
         # needed by the makefile - for these images, we only tag with commit sha
-        export BASE_IMAGE=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
-        export DOCKER_LABEL="${LABEL}"
-        export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
-        make vnc 
+        export EXTRA_DOCKER_LABEL="${LABEL}"
+        make $EXTENSION 
         docker push $DOCKER_PREFIX-$EXTENSION:$LABEL
 
   build-vnc-ext:
@@ -266,45 +264,13 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
         docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
-        
-        # needed by the makefile
-        export DOCKER_LABEL="${LABEL}"
-        export GIT_MASTER_HEAD_SHA=$(git rev-parse --short=7 --verify HEAD)
-        # the mainfile contains targets for each of these extensions; if we
-        # add another type of vnc based images, we need to add a target to
-        # the makrfile accordingly
+        docker pull ${DOCKER_PREFIX}-vnc:${LABEL}
+
+        # needed by the Makefile
+        export EXTRA_DOCKER_LABEL="${LABEL}"
         make vnc-$EXTENSION
         docker push $DOCKER_PREFIX-$EXTENSION:$LABEL
 
-  build-py-batch:
-    needs: build-py
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Docker Login
-      uses: Azure/docker-login@v1
-      with:
-        username: ${{ secrets.RENKU_DOCKER_USERNAME }}
-        password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
-
-    - uses: actions/checkout@v3
-    - name: Build renku project python-based docker image for batch execution
-      run: |
-        if [[ ${{ github.ref }} == refs/tags* ]]; then
-          export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
-        else
-          export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
-        fi
-        docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
-
-        # needed by the makefile
-        export DOCKER_LABEL="$LABEL"
-        export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
-        export RENKU_BASE="$DOCKER_PREFIX-py:${DEFAULT_PYTHON_VERSION}-$LABEL"
-        export BASE_IMAGE="python:3.10-slim-buster" 
-        make batch
-        echo "IMAGE_NAME=$DOCKER_PREFIX-batch:$DOCKER_LABEL" >> $GITHUB_OUTPUT
-
   build-julia-ext:
     needs: build-py
     runs-on: ubuntu-latest
@@ -334,12 +300,11 @@ jobs:
         docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
 
         # needed by the makefile
-        export RENKU_BASE="$DOCKER_PREFIX-py:${DEFAULT_PYTHON_VERSION}-$LABEL"
-        export DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL"
+        export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL"
         export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
         export JULIAVERSION="${{ matrix.JULIAVERSIONS }}"
         make julia
-        echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$DOCKER_LABEL" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT
         
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -392,12 +357,11 @@ jobs:
 
         # needed by the makefile
         export RVERSION="${{ matrix.RVERSION }}"
-        export DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL"
-        export RENKU_BASE=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-$LABEL
+        export R_DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL"
         export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" 
         export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}"
         make r
-        echo "IMAGE_NAME=$DOCKER_PREFIX-r:$DOCKER_LABEL" >> $GITHUB_OUTPUT
+        echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT
 
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v4
@@ -450,16 +414,15 @@ jobs:
         docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
 
         # needed by the makefile
-        export DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL"
-        export RENKU_BASE="$DOCKER_PREFIX-py:3.10-$LABEL"
+        export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL"
         export BIOC_VERSION="${{ matrix.RELEASE }}"
         make bioc
-        docker push $DOCKER_PREFIX-bioc:$DOCKER_LABEL
+        docker push $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL
 
         # on master push latest image
         if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ]
         then
-          docker tag $DOCKER_PREFIX-bioc:$DOCKER_TAG $DOCKER_PREFIX-bioc:latest
+          docker tag $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL $DOCKER_PREFIX-bioc:latest
           docker push $DOCKER_PREFIX-bioc:latest
         fi
 
diff --git a/Makefile b/Makefile
@@ -34,23 +34,23 @@ GIT_COMMIT_SHA?=$(shell git rev-parse --short=7 --verify HEAD)
 
 # for building the base image
 BASE_IMAGE_TAG?=lab-3.6.1
-RENKU_BASE?=$(DOCKER_PREFIX)-py:latest
-DOCKER_LABEL?=latest
+DEFAULT_PYTHON_VERSION?=3.10
+PY_DOCKER_LABEL?=$(DEFAULT_PYTHON_VERSION)-$(GIT_COMMIT_SHA)
+RENKU_BASE?=$(DOCKER_PREFIX)-py:$(PY_DOCKER_LABEL)
 
 # RStudio version
 RSTUDIO_VERSION_OVERRIDE?=2022.02.3-492
 RSTUDIO_BASE_IMAGE?=rocker/verse:devel
-
-# for building the r container
 RVERSION?=4.2.0
-TENSORFLOW_VERSION?=2.2.0
+R_DOCKER_LABEL?=$(RVERSION)-$(GIT_COMMIT_SHA)
 
 # for building the julia container
 JULIAVERSION?=1.7.1
+JULIA_DOCKER_LABEL?=$(JULIAVERSION)-$(GIT_COMMIT_SHA)
 
 # for building the bioconductor container
 BIOC_VERSION?=devel
-BIOC_TAG=$(BIOC_VERSION)
+BIOC_DOCKER_LABEL?=$(BIOC_VERSION)-$(GIT_COMMIT_SHA)
 
 # cuda defaults - these should be updated from time to time
 CUDA_BASE_IMAGE?=renku/renkulab-py:$(DOCKER_LABEL)
@@ -59,6 +59,10 @@ EXTRA_LIBRARIES?=
 CUDA_CUDART_PACKAGE?=cuda-cudart-11-7=11.7.60-1
 CUDA_COMPAT_PACKAGE?=cuda-compat-11-7
 LIBCUDNN_PACKAGE?=libcudnn8=8.5.0.96-1+cuda11.7
+CUDA_DOCKER_LABEL?=$(CUDA_VERSION)-$(GIT_COMMIT_SHA)
+
+# setup for the extras
+EXTRA_DOCKER_LABEL?=$(GIT_COMMIT_SHA)
 
 .PHONY: all
 
@@ -95,7 +99,7 @@ py:
 	docker build docker/py \
 		--build-arg BASE_IMAGE=jupyter/base-notebook:$(BASE_IMAGE_TAG) \
 		--platform=$(PLATFORM) \
-		-t $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)
+		-t $(DOCKER_PREFIX)-$@:$(PY_DOCKER_LABEL)
 
 r: py
 	docker build docker/r \
@@ -104,56 +108,49 @@ r: py
 		--build-arg RVERSION=$(RVERSION) \
 		--build-arg RSTUDIO_VERSION_OVERRIDE=$(RSTUDIO_VERSION_OVERRIDE) \
 		--platform=$(PLATFORM) \
-		-t $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)
+		-t $(DOCKER_PREFIX)-r:$(R_DOCKER_LABEL)
 
-# BASE_IMAGE was used for all the docker files, but if there are dependencies,
-# it can be expected to mean different things in different contexts; hence
-# CUDA_BASE_IMAGE was introduced here
 cuda: py
 	docker build docker/cuda \
-		--build-arg CUDA_BASE_IMAGE="$(CUDA_BASE_IMAGE)" \
+		--build-arg RENKU_BASE="$(RENKU_BASE)" \
 		--build-arg CUDA_COMPAT_PACKAGE="$(CUDA_COMPAT_PACKAGE)" \
 		--build-arg CUDA_CUDART_PACKAGE="$(CUDA_CUDART_PACKAGE)" \
 		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
 		--build-arg EXTRA_LIBRARIES="$(EXTRA_LIBRARIES)" \
 		--build-arg LIBCUDNN_PACKAGE="$(LIBCUDNN_PACKAGE)" \
 		--platform=$(PLATFORM) \
-		-t $(DOCKER_PREFIX)-cuda:$(DOCKER_LABEL)
-
-# this image is just tagged with the commit hash
-vnc: py
-	docker build docker/vnc \
-		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
-		-t $(DOCKER_PREFIX)-vnc:$(DOCKER_LABEL)
+		-t $(DOCKER_PREFIX)-cuda:$(CUDA_DOCKER_LABEL)
 
 # this image is tagged with the julia version and the commit hash
 julia: py
 	docker build docker/julia \
-		--build-arg BASE_IMAGE=$(RENKU_BASE) \
+		--build-arg RENKU_BASE=$(RENKU_BASE) \
 		--platform=$(PLATFORM) \
-		-t $(DOCKER_PREFIX)-julia:$(DOCKER_LABEL)
+		-t $(DOCKER_PREFIX)-julia:$(JULIA_DOCKER_LABEL)
 
-# this image is built on the vnc image and tagged as matlab with the commit hash
-vnc-matlab: vnc
-	docker build docker/matlab \
-		--build-arg BASE_IMAGE=renku/renkulab-vnc:$(GIT_COMMIT_SHA) \
-		-t $(DOCKER_PREFIX)-matlab:$(GIT_COMMIT_SHA)
+# this image is just tagged with the commit hash
+vnc: py
+	docker build docker/vnc \
+		--build-arg RENKU_BASE=$(RENKU_BASE) \
+		--platform=$(PLATFORM) \
+		-t $(DOCKER_PREFIX)-vnc:$(EXTRA_DOCKER_LABEL)
 
-vnc-qgis: vnc
-	docker build docker/qgis \
+# this image is built on the vnc image and tagged as matlab with the commit hash
+vnc-%: vnc
+	docker build docker/$* \
 		--build-arg BASE_IMAGE=renku/renkulab-vnc:$(GIT_COMMIT_SHA) \
 		--platform=$(PLATFORM) \
-		-t $(DOCKER_PREFIX)-qgis:$(GIT_COMMIT_SHA)
+		-t $(DOCKER_PREFIX)-$*:$(EXTRA_DOCKER_LABEL)
 
 batch: py
 	docker build docker/batch \
 		--build-arg RENKU_BASE="$(RENKU_BASE)" \
-		--build-arg BASE_IMAGE="python:3.9-slim-buster" \
-		-t $(DOCKER_PREFIX)-batch:$(GIT_COMMIT_SHA)
+		--build-arg BASE_IMAGE="python:3.10-slim-buster" \
+		-t $(DOCKER_PREFIX)-batch:$(EXTRA_DOCKER_LABEL)
 
 bioc: py
 	docker build docker/r \
 		--build-arg RENKU_BASE="$(RENKU_BASE)" \
 		--build-arg BASE_IMAGE="bioconductor/bioconductor_docker:$(BIOC_VERSION)" \
 		--platform=$(PLATFORM) \
-		-t $(DOCKER_PREFIX)-bioc:$(DOCKER_LABEL)
+		-t $(DOCKER_PREFIX)-bioc:$(BIOC_DOCKER_LABEL)
diff --git a/README.md b/README.md
@@ -7,7 +7,7 @@ environments like JupyterLab and RStudio from the Renku platform. They also each
 contain a version of the [renku cli](https://github.com/SwissDataScienceCenter/renku-python).
 
 The images are available on
-[DockerHub](https://hub.docker.com/search?q=renku%2Frenkulab-&type=image)and
+[DockerHub](https://hub.docker.com/search?q=renku%2Frenkulab-&type=image) and
 are automatically built from this repo via github actions (see the `.github`
 folder for more).
 
@@ -18,8 +18,8 @@ updating the `Makefile` to modify what it builds by default.
 
 ## Usage
 
-The basic python (renkulab-py), basic R (renkulab-r), and basic Julia (renkulab-julia)
-images are available via
+The basic python (`renkulab-py`), basic R (`renkulab-r`), and basic Julia (`renkulab-julia`)
+images are used in the 
 [renku project templates](https://github.com/SwissDataScienceCenter/renku-project-template)
 that you select upon renku project creation on the RenkuLab platform, or locally
 via `renku init`.
@@ -64,26 +64,21 @@ the image that's right for you:
 `renku/renkulab-[image flavor]:[image flavor version]-[tag|hash]`
 
 For example:
-`renku/renkulab-py:python-3.8.13-0.11.0`
+`renku/renkulab-py:python-3.10-0.15.0`
 
 * `renku/renkulab`: indicates this is an image you can use to spawn an environment
   from your project on RenkuLab.
 * `-py`: indicates this is a python image flavor; either the programming language
   installed in the environment, or the base image that extra dependencies are added to.
   See below for details about the available flavors.
-* `python-3.8.13`: indicates the version of python is 3.8.13
-* `0.11.0` (or `d572e9a`): the tag is a value given to a commit of the repository
+* `python-3.10`: indicates the version of python is 3.10
+* `0.15.0` (or `d572e9a`): the tag is a value given to a commit of the repository
   and indicates that the version is part of a release. If the version is not part of
   a release, this value is the first few chars of the git commit SHA from which the
   image is built.
 
-Note that the base images include the specified version of the `renku` CLI.
-This can easily be overridden by installing another version with `pipx` in the container
-or any Dockerfile that uses these images, e.g.
-
-```bash
-pipx install --force renku==<version>
-```
+Note that the base images include the latest version of the `renku` CLI.
+This can easily be overridden modifying the renku version in the project's Dockerfile.
 
 ## Current images
 
@@ -125,25 +120,19 @@ dockerhub: https://hub.docker.com/r/renku/renkulab-r/tags
 
 **Available via renku project templates**
 
-Based on the renkulab-py (python 3.9) image with julia installed.
+Based on the renkulab-py image with julia installed.
 
 dockerhub: https://hub.docker.com/r/renku/renkulab-julia/tags
 
 ### cuda
 
-Based on the renkulab-py with different versions of python and CUDA installed.
+Based on the `renkulab-py` with different versions of python and CUDA installed.
 
 dockerhub: https://hub.docker.com/r/renku/renkulab-cuda/tags
 
-### cuda-tf
-
-Based on the the cuda-tf images with different (compatible) versions of Tensorflow installed.
-
-dockerhub: https://hub.docker.com/r/renku/renkulab-cuda-tf/tags
-
 ### vnc
 
-Based on the renkulab-py (python 3.9) with a full virtual desktop installed.
+Based on the `renkulab-py` image with a full virtual desktop installed.
 It uses noVNC 1.1.0 and TigerVNC 1.9.0 with a Renku UI to deliver a Linux desktop.
 
 https://hub.docker.com/r/renku/renkulab-vnc/tags
@@ -156,7 +145,7 @@ https://hub.docker.com/r/renku/renkulab-matlab/tags
 
 ### qgis
 
-A full virtual desktop as above with QGIS  installed.
+A full virtual desktop as above with QGIS installed.
 
 https://hub.docker.com/r/renku/renkulab-qgis/tags
 
@@ -175,7 +164,7 @@ It may be necessary to build individual images directly with `docker`; this
 is done by running `docker build -t <name:tag> .` in the directory of the 
 image you would like to build. Note that on arm-based systems (e.g. Apple 
 M1/M2) you may need to use the flag `--platform=linux/amd64` for the
-build because `git-lfs 3.2.0` is not available for the arm architecture.
+build because not all base images are available for ARM architecture. 
 
 ## Adding renku to your own images
 

diff --git a/docker/cuda/Dockerfile b/docker/cuda/Dockerfile
@@ -1,5 +1,5 @@
-ARG CUDA_BASE_IMAGE=renku/renkulab-py:latest
-FROM $CUDA_BASE_IMAGE
+ARG RENKU_BASE=renku/renkulab-py:latest
+FROM $RENKU_BASE
 
 LABEL maintainer="Swiss Data Science Center <[email protected]>"
 

diff --git a/docker/julia/Dockerfile b/docker/julia/Dockerfile
@@ -1,5 +1,5 @@
-ARG BASE_IMAGE=renku/renkulab-py:latest
-FROM $BASE_IMAGE as base
+ARG RENKU_BASE=renku/renkulab-py:latest
+FROM $RENKU_BASE as base
 
 LABEL maintainer="Swiss Data Science Center <[email protected]>"
 

diff --git a/docker/r/Dockerfile b/docker/r/Dockerfile
@@ -1,5 +1,5 @@
 # define build arguments
-ARG RENKU_BASE=renku/renkulab-py:3.9-d7394dd
+ARG RENKU_BASE=renku/renkulab-py:latest
 ARG BASE_IMAGE=rocker/verse:devel
 
 # define base images