Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: image tagging consistency #313

Merged
merged 7 commits into from
Mar 9, 2023
Merged

fix: image tagging consistency #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8253905
fix: ensure consistency with DOCKER_LABEL
rokroskar Mar 1, 2023
8768c2b
chore: pull base image before build
rokroskar Mar 1, 2023
f68b2bc
Merge branch 'master' into 000-fix-build
rokroskar Mar 1, 2023
d671a5a
chore: fix julia build
rokroskar Mar 1, 2023
50e5c68
chore: remove unneeded env var
rokroskar Mar 1, 2023
f7fa4c6
chore: fix up naming and variables
rokroskar Mar 1, 2023
e417be8
chore: remove problematic libs from matlab
rokroskar Mar 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 41 additions & 89 deletions .github/workflows/build-and-push-to-docker.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,10 @@ name: Renku Docker Image CI
on: [push]

env:
DOCKER_NAME: "renku/renkulab"
DOCKER_PREFIX: renku/renkulab
DEFAULT_PYTHON_VERSION: "3.10"

jobs:

lint:
strategy:
fail-fast: true
Expand Down Expand Up @@ -75,8 +74,6 @@ jobs:
- uses: actions/checkout@v3
- name: Build renku project python-based docker images
id: build-image
env:
DOCKER_PREFIX: renku/renkulab
run: |
if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
Expand All @@ -94,10 +91,10 @@ jobs:
# needed by the makefile - these are generated dynamically
# and hence they are not part of the env setup above
export BASE_IMAGE_TAG=${{ matrix.BASE_IMAGE_TAG }}
export DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}"
export PY_DOCKER_LABEL="${RENKU_PYTHON_BASE_IMAGE_TAG}-${LABEL}"
export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
make py
echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${DOCKER_LABEL}" >> $GITHUB_OUTPUT
echo "IMAGE_NAME=${DOCKER_PREFIX}-py:${PY_DOCKER_LABEL}" >> $GITHUB_OUTPUT

- name: Image Acceptance Tests
uses: cypress-io/github-action@v4
Expand All @@ -115,8 +112,8 @@ jobs:

# on master push latest image - the lab-* image is tagged latest
if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.BASE_IMAGE_TAG }}" == lab-* ]; then
docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_NAME-py:latest
docker push $DOCKER_NAME-py:latest
docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-py:latest
docker push $DOCKER_PREFIX-py:latest
fi

build-cuda:
Expand Down Expand Up @@ -163,30 +160,26 @@ jobs:
username: ${{ secrets.RENKU_DOCKER_USERNAME }}
password: ${{ secrets.RENKU_DOCKER_PASSWORD }}

- uses: actions/checkout@v3
- uses: actions/checkout@v3
- name: Build renku project python-based docker image extensions
id: build-image
env:
DOCKER_PREFIX: "renku/renkulab"
run: |
# cd docker/cuda

if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
else
export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
fi
docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}

# these are set dynamically so not part of env
export DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}"
export CUDA_BASE_IMAGE="renku/renkulab-py:$DEFAULT_PYTHON_VERSION-$LABEL"
export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}"
export CUDA_CUDART_PACKAGE="${{ matrix.CUDA_CUDART_PACKAGE }}"
export CUDA_VERSION="${{ matrix.CUDA_VERSION }}"
export CUDA_DOCKER_LABEL="${{ matrix.CUDA_VERSION }}-${LABEL}"
export EXTRA_LIBRARIES="${{ matrix.EXTRA_LIBRARIES }}"
export CUDA_CUDART_PACKAGE="${{ matrix.CUDA_CUDART_PACKAGE }}"
export CUDA_COMPAT_PACKAGE="${{ matrix.CUDA_COMPAT_PACKAGE }}"
export LIBCUDNN_PACKAGE="${{ matrix.LIBCUDNN_PACKAGE }}"
make cuda
echo "IMAGE_NAME=$DOCKER_NAME-cuda:$DOCKER_LABEL" >> $GITHUB_OUTPUT
echo "IMAGE_NAME=$DOCKER_PREFIX-cuda:$CUDA_DOCKER_LABEL" >> $GITHUB_OUTPUT

- name: Image Acceptance Tests
uses: cypress-io/github-action@v4
Expand All @@ -202,9 +195,9 @@ jobs:
docker push ${{ steps.build-image.outputs.IMAGE_NAME }}

# on master push latest image
if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.CUDA_VERSION }}" == "11.5" ]; then
docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_NAME-cuda:latest
docker push $DOCKER_NAME-cuda:latest
if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.CUDA_VERSION }}" == "11.7" ]; then
docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-cuda:latest
docker push $DOCKER_PREFIX-cuda:latest
fi

build-py-ext:
Expand All @@ -216,6 +209,7 @@ jobs:
matrix:
EXTENSIONS:
- vnc
- batch

steps:
- name: Docker Login
Expand All @@ -227,20 +221,19 @@ jobs:
- uses: actions/checkout@v3
- name: Build renku project python-based docker image extensions
env:
DOCKER_PREFIX: renku/renkulab
EXTENSION: ${{ matrix.EXTENSIONS }}
run: |
if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
else
export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
fi

docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}

# needed by the makefile - for these images, we only tag with commit sha
export DOCKER_LABEL="${LABEL}"
export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
make vnc
docker push $DOCKER_NAME-$EXTENSION:$LABEL
export EXTRA_DOCKER_LABEL="${LABEL}"
make $EXTENSION
docker push $DOCKER_PREFIX-$EXTENSION:$LABEL

build-vnc-ext:
needs: build-py-ext
Expand All @@ -263,54 +256,20 @@ jobs:
- uses: actions/checkout@v3
- name: Build renku project python-based docker image extensions
env:
DOCKER_PREFIX: renku/renkulab
EXTENSION: ${{ matrix.VNC-EXTENSIONS }}
DOCKER_NAME: "renku/renkulab"
run: |
if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
else
export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
fi
docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}
docker pull ${DOCKER_PREFIX}-vnc:${LABEL}

# needed by the makefile
export DOCKER_LABEL="${LABEL}"
export GIT_MASTER_HEAD_SHA=$(git rev-parse --short=7 --verify HEAD)
# the mainfile contains targets for each of these extensions; if we
# add another type of vnc based images, we need to add a target to
# the makrfile accordingly
# needed by the Makefile
export EXTRA_DOCKER_LABEL="${LABEL}"
make vnc-$EXTENSION
docker push $DOCKER_NAME-$EXTENSION:$LABEL

build-py-batch:
needs: build-py
runs-on: ubuntu-latest

steps:
- name: Docker Login
uses: Azure/docker-login@v1
with:
username: ${{ secrets.RENKU_DOCKER_USERNAME }}
password: ${{ secrets.RENKU_DOCKER_PASSWORD }}

- uses: actions/checkout@v3
- name: Build renku project python-based docker image for batch execution
env:
DOCKER_PREFIX: renku/renkulab
run: |
if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
else
export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
fi

# needed by the makefile
export DOCKER_LABEL="$LABEL"
export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
export RENKU_BASE="$DOCKER_PREFIX-py:3.10-$LABEL"
export BASE_IMAGE="python:3.10-slim-buster"
make batch
echo "IMAGE_NAME=$DOCKER_NAME-batch:$DOCKER_LABEL" >> $GITHUB_OUTPUT
docker push $DOCKER_PREFIX-$EXTENSION:$LABEL

build-julia-ext:
needs: build-py
Expand All @@ -332,22 +291,20 @@ jobs:
- uses: actions/checkout@v3
- name: Build renku project julia docker image extensions
id: build-image
env:
DOCKER_PREFIX: renku/renkulab
run: |
if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
else
export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
fi
docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}

# needed by the makefile
export DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL"
export JULIAVERSION="${{ matrix.JULIAVERSIONS }}"
export JULIA_DOCKER_LABEL="${{ matrix.JULIAVERSIONS }}-$LABEL"
export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
export BASE_IMAGE="$DOCKER_NAME-py:3.10-$LABEL"
export JULIAVERSION="${{ matrix.JULIAVERSIONS }}"
make julia
echo "IMAGE_NAME=$DOCKER_NAME-julia:$DOCKER_LABEL" >> $GITHUB_OUTPUT
echo "IMAGE_NAME=$DOCKER_PREFIX-julia:$JULIA_DOCKER_LABEL" >> $GITHUB_OUTPUT

- name: Image Acceptance Tests
uses: cypress-io/github-action@v4
Expand All @@ -361,7 +318,7 @@ jobs:
- name: Push to docker
run: |
docker push ${{ steps.build-image.outputs.IMAGE_NAME }}

build-r-ubuntu:
needs: build-py
runs-on: ubuntu-latest
Expand Down Expand Up @@ -390,24 +347,21 @@ jobs:
- uses: actions/checkout@v3
- name: Build renku project rocker docker images
id: build-image
env:
DOCKER_PREFIX: renku/renkulab
run: |
if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
else
export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
fi
docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}

# needed by the makefile
export RVERSION="${{ matrix.RVERSION }}"
export DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL"
export RENKU_PYTHON_BASE_IMAGE_TAG="3.10"
export GIT_COMMIT_SHA=$(git rev-parse --short=7 --verify HEAD)
export BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}"
export R_DOCKER_LABEL="${{ matrix.RVERSION }}-$LABEL"
export RSTUDIO_BASE_IMAGE="rocker/verse:${{ matrix.RVERSION }}"
export RSTUDIO_VERSION_OVERRIDE="${{ matrix.RSTUDIO_VERSION }}"
make r
echo "IMAGE_NAME=$DOCKER_PREFIX-r:$DOCKER_LABEL" >> $GITHUB_OUTPUT
echo "IMAGE_NAME=$DOCKER_PREFIX-r:$R_DOCKER_LABEL" >> $GITHUB_OUTPUT

- name: Image Acceptance Tests
uses: cypress-io/github-action@v4
Expand All @@ -427,8 +381,8 @@ jobs:
# on master push latest image
if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RVERSION }}" == "4.2.0" ]
then
docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_NAME-r:latest
docker push $DOCKER_NAME-r:latest
docker tag ${{ steps.build-image.outputs.IMAGE_NAME }} $DOCKER_PREFIX-r:latest
docker push $DOCKER_PREFIX-r:latest
fi

build-bioc:
Expand All @@ -451,26 +405,24 @@ jobs:

- uses: actions/checkout@v3
- name: Build renku project bioconductor docker images
env:
DOCKER_PREFIX: renku/renkulab
run: |
if [[ ${{ github.ref }} == refs/tags* ]]; then
export LABEL=$(echo ${{ github.ref }} | cut -d / -f 3)
else
export LABEL=$(echo ${{ github.sha }} | cut -c 1-7)
fi
docker pull ${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${LABEL}

# needed by the makefile
export DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL"
export RENKU_BASE="$DOCKER_PREFIX-py:3.10-$LABEL"
export BIOC_DOCKER_LABEL="${{ matrix.RELEASE }}-$LABEL"
export BIOC_VERSION="${{ matrix.RELEASE }}"
make bioc
docker push $DOCKER_PREFIX-bioc:$DOCKER_LABEL
docker push $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL

# on master push latest image
if [ "$REF" == "refs/heads/master" ] && [ "${{ matrix.RELEASE }}" == "devel" ]
then
docker tag $DOCKER_NAME-bioc:$DOCKER_TAG $DOCKER_NAME-bioc:latest
docker push $DOCKER_NAME-bioc:latest
docker tag $DOCKER_PREFIX-bioc:$BIOC_DOCKER_LABEL $DOCKER_PREFIX-bioc:latest
docker push $DOCKER_PREFIX-bioc:latest
fi

Loading