diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml new file mode 100644 index 00000000..d5767905 --- /dev/null +++ b/.github/workflows/acceptance_tests.yml @@ -0,0 +1,78 @@ +name: Acceptance tests + +on: + pull_request: + types: + - synchronize + +env: + DOCKER_PLATFORMS: linux/amd64,linux/arm64 + +jobs: + build-base-python-image: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + - name: Build Image + run: | + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t py docker/py/ + docker save py > /tmp/py.tar + - name: Upload Image Artifact + uses: actions/upload-artifact@v2 + with: + name: py + path: /tmp/py.tar + + acceptance-tests: + needs: build-base-python-image + strategy: + matrix: + include: + - image_name: py + test_user: jovyan + - image_name: cuda-tf + build_arg: BASE_IMAGE=py + test_user: jovyan + - image_name: generic + build_arg: RENKU_BASE=py + test_user: jovyan + - image_name: julia + build_arg: BASE_IMAGE=py + test_user: jovyan + - image_name: r + build_arg: RENKU_BASE=py + test_user: rstudio + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Download Base Image + uses: actions/download-artifact@v2 + with: + name: py + path: /tmp + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 + - name: Load Base Image in Docker + run: | + docker load --input /tmp/py.tar + - name: Build images + if: ${{ matrix.image_name != 'py' }} + run: | + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} -t ${{ matrix.image_name }} --build-arg ${{ matrix.build_arg }} docker/${{ matrix.image_name }}/ + - name: Py Image Acceptance Tests + uses: cypress-io/github-action@v2 + env: + TEST_IMAGE_NAME: ${{ matrix.image_name }} + TEST_USER_NAME: ${{ matrix.test_user }} + with: + working-directory: tests + command: npx mocha test.js diff --git a/.github/workflows/build_and_push_to_docker.yml b/.github/workflows/build_and_push_to_docker.yml index 6d0346c3..1e51cf4f 100644 --- a/.github/workflows/build_and_push_to_docker.yml +++ b/.github/workflows/build_and_push_to_docker.yml @@ -4,6 +4,7 @@ on: [push] env: DOCKER_NAME: "renku/renkulab" + DOCKER_PLATFORMS: linux/amd64,linux/arm64 jobs: @@ -28,6 +29,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project python-based docker images id: build-image run: | @@ -44,7 +50,12 @@ jobs: export LABEL_PREFIX=${{ matrix.BASE_IMAGE_TAG }} fi - docker build docker/py --tag $DOCKER_NAME-py:$LABEL_PREFIX-$LABEL --build-arg BASE_IMAGE="jupyter/base-notebook:${{ matrix.BASE_IMAGE_TAG }}" + docker buildx build \ + --platform ${{env.DOCKER_PLATFORMS}} \ + --tag $DOCKER_NAME-py:$LABEL_PREFIX-$LABEL \ + --build-arg BASE_IMAGE="jupyter/base-notebook:${{ matrix.BASE_IMAGE_TAG }}" \ + --push \ + docker/py echo "::set-output name=IMAGE_NAME::$DOCKER_NAME-py:$LABEL_PREFIX-$LABEL" - name: Image Acceptance Tests uses: cypress-io/github-action@v2 @@ -82,6 +93,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project python-based docker image extensions run: | if [[ ${{ github.ref }} == refs/tags* ]]; then @@ -90,7 +106,7 @@ jobs: export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker build docker/$EXTENSION \ + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/$EXTENSION \ --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \ --tag $DOCKER_NAME-$EXTENSION:$LABEL docker push $DOCKER_NAME-$EXTENSION:$LABEL @@ -268,12 +284,12 @@ jobs: docker build docker/$EXTENSION \ --build-arg BASE_IMAGE="$DOCKER_NAME-vnc:$LABEL" \ - --tag $DOCKER_NAME-$EXTENSION:$LABEL - docker push $DOCKER_NAME-$EXTENSION:$LABEL + --tag $DOCKER_NAME-$EXTENSION:$LABEL \ + --push env: EXTENSION: ${{ matrix.VNC-EXTENSIONS }} DOCKER_NAME: "renku/renkulab" - + build-py-batch: needs: build-py runs-on: ubuntu-latest @@ -286,6 +302,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project python-based docker image for batch execution run: | if [[ ${{ github.ref }} == refs/tags* ]]; then @@ -294,11 +315,11 @@ jobs: export LABEL=$(echo ${{ github.sha }} | cut -c 1-7) fi - docker build docker/batch \ + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/batch \ --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \ --build-arg BASE_IMAGE="python:3.9-slim-buster" \ - --tag $DOCKER_NAME-batch:$LABEL - docker push $DOCKER_NAME-batch:$LABEL + --tag $DOCKER_NAME-batch:$LABEL \ + --push env: DOCKER_NAME: "renku/renkulab" @@ -320,6 +341,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project julia docker image extensions id: build-image run: | @@ -330,7 +356,7 @@ jobs: fi export DOCKER_TAG="${{ matrix.JULIAVERSIONS }}-$LABEL" - docker build docker/julia \ + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/julia \ --build-arg BASE_IMAGE="$DOCKER_NAME-py:3.9-$LABEL" \ --tag $DOCKER_NAME-julia:$DOCKER_TAG echo "::set-output name=IMAGE_NAME::$DOCKER_NAME-julia:$DOCKER_TAG" @@ -367,6 +393,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project rocker docker images id: build-image run: | @@ -377,11 +408,11 @@ jobs: fi export DOCKER_TAG="${{ matrix.RVERSION }}-$LABEL" - docker build docker/r \ + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \ --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \ --build-arg BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}" \ --tag $DOCKER_NAME-r:$DOCKER_TAG - + echo "::set-output name=IMAGE_NAME::$DOCKER_NAME-r:$DOCKER_TAG" - name: Image Acceptance Tests uses: cypress-io/github-action@v2 @@ -421,6 +452,11 @@ jobs: password: ${{ secrets.RENKU_DOCKER_PASSWORD }} - uses: actions/checkout@v2 + - name: Set up QEMU + # https://github.com/docker/setup-qemu-action/releases/tag/v1.2.0 + uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 - name: Build renku project bioconductor docker images run: | if [[ ${{ github.ref }} == refs/tags* ]]; then @@ -430,7 +466,7 @@ jobs: fi export DOCKER_TAG="${{ matrix.RELEASE }}-$LABEL" - docker build docker/r \ + docker buildx build --platform ${{env.DOCKER_PLATFORMS}} docker/r \ --build-arg RENKU_BASE="$DOCKER_NAME-py:3.9-$LABEL" \ --build-arg BASE_IMAGE=bioconductor/bioconductor_docker:${{ matrix.RELEASE }} \ --tag $DOCKER_NAME-bioc:$DOCKER_TAG diff --git a/Makefile b/Makefile index dc7567f0..4ea07b07 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ extensions = \ DOCKER_PREFIX?=renku/renkulab DOCKER_LABEL?=latest +PLATFORMS?=linux/amd64,linux/arm64 GIT_MASTER_HEAD_SHA:=$(shell git rev-parse --short=7 --verify HEAD) # for the empty version case: @@ -66,14 +67,14 @@ pull: done r: py - docker build docker/r \ + docker buildx build --platform $(PLATFORMS) docker/r \ --build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \ --build-arg RVERSION=$(RVERSION) \ -t $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) && \ docker tag $(DOCKER_PREFIX)-r:$(DOCKER_LABEL)$(RENKU_TAG)$(R_TAG) $(DOCKER_PREFIX)-r:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)$(R_TAG) bioc: py - docker build docker/bioc \ + docker buildx build --platform $(PLATFORMS) docker/bioc \ --build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \ --build-arg RELEASE=$(BIOC_VERSION) \ -t $(DOCKER_PREFIX)-bioc:$(DOCKER_LABEL)$(RENKU_TAG)$(BIOC_TAG) && \ @@ -82,26 +83,26 @@ bioc: py py: cd docker/$@ && \ - docker build \ + docker buildx build --platform $(PLATFORMS) \ -t $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) . && \ docker tag $(DOCKER_PREFIX)-$@:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-$@:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) cuda: py - docker build docker/cuda-tf \ + docker buildx build --platform $(PLATFORMS) docker/cuda-tf \ --build-arg RENKU_PIP_SPEC=$(RENKU_PIP_SPEC) \ --build-arg RENKU_BASE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA) \ --build-arg TENSORFLOW_VERSION=$(TENSORFLOW_VERSION) \ -t $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) && \ docker tag $(DOCKER_PREFIX)-cuda-tf:$(DOCKER_LABEL) $(DOCKER_PREFIX)-cuda-tf:$(GIT_MASTER_HEAD_SHA) - + vnc: py - docker build docker/vnc \ + docker buildx build --platform $(PLATFORMS) docker/vnc \ --build-arg BASE_IMAGE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \ -t $(DOCKER_PREFIX)-vnc:$(DOCKER_LABEL)$(RENKU_TAG) && \ docker tag $(DOCKER_PREFIX)-vnc:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-vnc:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) julia: py - docker build docker/julia \ + docker buildx build --platform $(PLATFORMS) docker/julia \ --build-arg BASE_IMAGE=renku/renkulab-py:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG) \ -t $(DOCKER_PREFIX)-julia:$(DOCKER_LABEL)$(RENKU_TAG) && \ docker tag $(DOCKER_PREFIX)-julia:$(DOCKER_LABEL)$(RENKU_TAG) $(DOCKER_PREFIX)-julia:$(GIT_MASTER_HEAD_SHA)$(RENKU_TAG)