support multi-platform build using buildx

SwissDataScienceCenter · Jun 14, 2022 · 3b491a6 · 3b491a6
1 parent e8f95bf
commit 3b491a6
Show file tree

Hide file tree

Showing 3 changed files with 134 additions and 19 deletions.
diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml
@@ -0,0 +1,78 @@
+name: Acceptance tests
+
+on:
+  pull_request:
+    types:
+    - synchronize
+
+env:
+  DOCKER_PLATFORMS: linux/amd64,linux/arm64
+
+jobs:
+  build-base-python-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up QEMU
+        # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
+      - name: Build Image
+        run: |
+          docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t py docker/py/
+          docker save py > /tmp/py.tar
+      - name: Upload Image Artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: py
+          path: /tmp/py.tar
+
+  acceptance-tests:
+    needs: build-base-python-image
+    strategy:
+      matrix:
+        include:
+          - image_name: py
+            test_user: jovyan
+          - image_name: cuda-tf
+            build_arg: BASE_IMAGE=py
+            test_user: jovyan
+          - image_name: generic
+            build_arg: RENKU_BASE=py
+            test_user: jovyan
+          - image_name: julia
+            build_arg: BASE_IMAGE=py
+            test_user: jovyan
+          - image_name: r
+            build_arg: RENKU_BASE=py
+            test_user: rstudio
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Download Base Image
+        uses: actions/download-artifact@v2
+        with:
+          name: py
+          path: /tmp
+      - name: Set up QEMU
+        # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
+      - name: Load Base Image in Docker
+        run: |
+          docker load --input /tmp/py.tar
+      - name: Build images
+        if: ${{ matrix.image_name != 'py' }}
+        run: |
+          docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/
+      - name: Py Image Acceptance Tests
+        uses: cypress-io/github-action@v2
+        env:
+          TEST_IMAGE_NAME: ${{ matrix.image_name }}
+          TEST_USER_NAME: ${{ matrix.test_user }}
+        with:
+          working-directory: tests
+          command: npx mocha test.js
diff --git a/.github/workflows/build_and_push_to_docker.yml b/.github/workflows/build_and_push_to_docker.yml
@@ -4,6 +4,7 @@ on: [push]
 
 env:
   DOCKER_NAME: "renku/renkulab"
+  DOCKER_PLATFORMS: linux/amd64,linux/arm64
 
 jobs:
 
@@ -28,6 +29,11 @@ jobs:
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project python-based docker images
       id: build-image
       run: |
@@ -44,7 +50,12 @@ jobs:
           export LABEL_PREFIX=${{ matrix.BASE_IMAGE_TAG }}
         fi
 
-        docker build docker/py --tag $DOCKER_NAME-py:$LABEL_PREFIX-$LABEL --build-arg BASE_IMAGE="jupyter/base-notebook:${{ matrix.BASE_IMAGE_TAG }}"
+        docker buildx build \
+          --platform ${{env.DOCKER_PLATFORMS}} \
+          --tag $DOCKER_NAME-py:$LABEL_PREFIX-$LABEL \
+          --build-arg BASE_IMAGE="jupyter/base-notebook:${{ matrix.BASE_IMAGE_TAG }}" \
+          --push \
+          docker/py
         echo "::set-output name=IMAGE_NAME::$DOCKER_NAME-py:$LABEL_PREFIX-$LABEL"
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v2
@@ -82,6 +93,11 @@ jobs:
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project python-based docker image extensions
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -90,7 +106,7 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker build docker/$EXTENSION \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/$EXTENSION \
           --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \
           --tag $DOCKER_NAME-$EXTENSION:$LABEL
         docker push $DOCKER_NAME-$EXTENSION:$LABEL
@@ -268,12 +284,12 @@ jobs:
 
         docker build docker/$EXTENSION \
           --build-arg BASE_IMAGE="$DOCKER_NAME-vnc:$LABEL" \
-          --tag $DOCKER_NAME-$EXTENSION:$LABEL
-        docker push $DOCKER_NAME-$EXTENSION:$LABEL
+          --tag $DOCKER_NAME-$EXTENSION:$LABEL \
+          --push
       env:
         EXTENSION: ${{ matrix.VNC-EXTENSIONS }}
         DOCKER_NAME: "renku/renkulab"
-  
+
   build-py-batch:
     needs: build-py
     runs-on: ubuntu-latest
@@ -286,6 +302,11 @@ jobs:
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project python-based docker image for batch execution
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -294,11 +315,11 @@ jobs:
           export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
         fi
 
-        docker build docker/batch \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/batch \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE="python:3.9-slim-buster" \
-          --tag $DOCKER_NAME-batch:$LABEL
-        docker push $DOCKER_NAME-batch:$LABEL
+          --tag $DOCKER_NAME-batch:$LABEL \
+          --push
       env:
         DOCKER_NAME: "renku/renkulab"
 
@@ -320,6 +341,11 @@ jobs:
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project julia docker image extensions
       id: build-image
       run: |
@@ -330,7 +356,7 @@ jobs:
         fi
 
         export DOCKER_TAG="${{ matrix.JULIAVERSIONS }}-$LABEL"
-        docker build docker/julia \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/julia \
           --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \
           --tag $DOCKER_NAME-julia:$DOCKER_TAG
         echo "::set-output name=IMAGE_NAME::$DOCKER_NAME-julia:$DOCKER_TAG"
@@ -367,6 +393,11 @@ jobs:
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project rocker docker images
       id: build-image
       run: |
@@ -377,11 +408,11 @@ jobs:
         fi
 
         export DOCKER_TAG="${{ matrix.RVERSION }}-$LABEL"
-        docker build docker/r \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" \
           --tag $DOCKER_NAME-r:$DOCKER_TAG
-        
+
         echo "::set-output name=IMAGE_NAME::$DOCKER_NAME-r:$DOCKER_TAG"
     - name: Image Acceptance Tests
       uses: cypress-io/github-action@v2
@@ -421,6 +452,11 @@ jobs:
         password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
 
     - uses: actions/checkout@v2
+    - name: Set up QEMU
+      # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0
+      uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
     - name: Build renku project bioconductor docker images
       run: |
         if [[ ${{ github.ref }} == refs/tags* ]]; then
@@ -430,7 +466,7 @@ jobs:
         fi
 
         export DOCKER_TAG="${{ matrix.RELEASE }}-$LABEL"
-        docker build docker/r \
+        docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \
           --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \
           --build-arg BASE_IMAGE=bioconductor/bioconductor_docker:${{ matrix.RELEASE }} \
           --tag $DOCKER_NAME-bioc:$DOCKER_TAG

diff --git a/Makefile b/Makefile
@@ -26,6 +26,7 @@ extensions = \
 
 DOCKER_PREFIX?=renku/renkulab
 DOCKER_LABEL?=latest
+PLATFORMS?=linux/amd64,linux/arm64
 GIT_MASTER_HEAD_SHA:=$(shell git rev-parse --short=7 --verify HEAD)
 
 # for the empty version case:
@@ -66,14 +67,14 @@ pull:
 	done
 
 r: py
-	docker build docker/r \
+	docker buildx build --platform $(PLATFORMS) docker/r \
 		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		--build-arg RVERSION=$(RVERSION) \
 		-t $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) && \
 	docker tag $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) $(DOCKER_PREFIX)-r:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)$(R_TAG)
 
 bioc: py
-	docker build docker/bioc \
+	docker buildx build --platform $(PLATFORMS) docker/bioc \
 		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		--build-arg RELEASE=$(BIOC_VERSION) \
 		-t $(DOCKER_PREFIX)-bioc:$(DOCKER_LABEL)$(RENKU_TAG)$(BIOC_TAG) && \
@@ -82,26 +83,26 @@ bioc: py
 
 py:
 	cd docker/$@ && \
-	docker build \
+	docker buildx build --platform $(PLATFORMS) \
 		-t $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) . && \
 	docker tag $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-$@:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)
 
 cuda: py
-	docker build docker/cuda-tf \
+	docker buildx build --platform $(PLATFORMS) docker/cuda-tf \
 		--build-arg RENKU_PIP_SPEC=$(RENKU_PIP_SPEC) \
 		--build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA) \
 		--build-arg TENSORFLOW_VERSION=$(TENSORFLOW_VERSION) \
 		-t $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) && \
 	docker tag $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) $(DOCKER_PREFIX)-cuda-tf:$(GIT_MASTER_HEAD_SHA)
-	
+
 vnc: py
-	docker build docker/vnc \
+	docker buildx build --platform $(PLATFORMS) docker/vnc \
 		--build-arg BASE_IMAGE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		-t $(DOCKER_PREFIX)-vnc:$(DOCKER_LABEL)$(RENKU_TAG) && \
 	docker tag $(DOCKER_PREFIX)-vnc:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-vnc:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)
 
 julia: py
-	docker build docker/julia \
+	docker buildx build --platform $(PLATFORMS) docker/julia \
 		--build-arg BASE_IMAGE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \
 		-t $(DOCKER_PREFIX)-julia:$(DOCKER_LABEL)$(RENKU_TAG) && \
 	docker tag $(DOCKER_PREFIX)-julia:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-julia:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)