Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: image with conda packages in homedir #391

Merged
merged 4 commits into from
Oct 17, 2023
Merged

feat: image with conda packages in homedir #391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3ad0e8c
feat: image with conda packages in homedir
olevski Oct 16, 2023
d051c13
squashme: minor fix
olevski Oct 17, 2023
48961fc
squashme: update docker build actions
olevski Oct 17, 2023
dc9f02f
chore: make disk space
olevski Oct 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 90 additions & 18 deletions .github/workflows/build-and-push-to-docker.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ jobs:
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
uses: docker/setup-buildx-action@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
Expand All @@ -80,7 +80,7 @@ jobs:
tags: |
type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}-
- name: Build and load
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}
Expand All @@ -103,7 +103,7 @@ jobs:
# export the default base image for other jobs to use
- name: Build and export
if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}
Expand All @@ -114,6 +114,72 @@ jobs:
tags: ${{ steps.meta.outputs.tags }}
provenance: false

build-py-homedir-env:
needs: lint
runs-on: ubuntu-latest

strategy:
fail-fast: true
matrix:
BASE_PYTHON_VERSION:
- "3.10"
- "3.9"
- "3.8"
steps:
- name: Docker Login
uses: Azure/docker-login@v1
with:
username: ${{ secrets.RENKU_DOCKER_USERNAME }}
password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
- uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
with:
images: ${{ env.DOCKER_PREFIX }}-py-homedir-envs
tags: |
type=sha,prefix=${{ matrix.BASE_PYTHON_VERSION }}-
- name: Build and load
uses: docker/build-push-action@v5
with:
build-args: |
BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}
CONDA_ENVS_DIRS=/home/jovyan/work/envs
cache-from: type=gha
cache-to: type=gha,mode=max
context: docker/py
labels: ${{ steps.meta.outputs.labels }}
load: true
tags: ${{ steps.meta.outputs.tags }}
provenance: false
- name: Image Acceptance Tests
uses: cypress-io/github-action@v4
env:
TEST_IMAGE_NAME: ${{ env.DOCKER_PREFIX }}-py-homedir-envs:${{ steps.meta.outputs.version }}
TEST_USER_NAME: jovyan
TEST_SPEC: jupyterlab.cy.ts
with:
working-directory: tests
command: npx mocha -r ts-node/register index.ts
# export the default base image for other jobs to use
- name: Build and export
if: matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION
uses: docker/build-push-action@v5
with:
build-args: |
BASE_IMAGE=jupyter/base-notebook:python-${{ matrix.BASE_PYTHON_VERSION }}
CONDA_ENVS_DIRS=/home/jovyan/work/envs
cache-from: type=gha
context: docker/py
labels: ${{ steps.meta.outputs.labels }}
push: true
tags: ${{ steps.meta.outputs.tags }}
provenance: false

build-py-multiarch:
needs: build-py
runs-on: ubuntu-latest
Expand All @@ -135,7 +201,7 @@ jobs:
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
uses: docker/setup-buildx-action@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
Expand All @@ -146,7 +212,7 @@ jobs:
type=semver,pattern={{version}},prefix=${{ matrix.BASE_PYTHON_VERSION }}-
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.BASE_PYTHON_VERSION == env.DEFAULT_PYTHON_VERSION }}
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
context: docker/py
push: true
Expand Down Expand Up @@ -213,7 +279,7 @@ jobs:
echo "sha_short=$sha_short" >> $GITHUB_OUTPUT
echo "renku_base=${DOCKER_PREFIX}-py:${DEFAULT_PYTHON_VERSION}-${sha_short}" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
uses: docker/setup-buildx-action@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
Expand All @@ -224,7 +290,7 @@ jobs:
type=semver,pattern={{version}},prefix=${{ matrix.CUDA_VERSION }}-
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.CUDA_VERSION == '11.7' }}
- name: Build and load
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
RENKU_BASE=${{ steps.vars.outputs.renku_base }}
Expand Down Expand Up @@ -252,7 +318,7 @@ jobs:
working-directory: tests
command: npx mocha -r ts-node/register index.ts
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
RENKU_BASE=${{ steps.vars.outputs.renku_base }}
Expand Down Expand Up @@ -295,7 +361,7 @@ jobs:
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
uses: docker/setup-buildx-action@v3
- name: Docker meta
id: meta
uses: docker/metadata-action@v4
Expand All @@ -306,7 +372,7 @@ jobs:
type=semver,pattern={{version}},prefix=
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }},prefix=
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
RENKU_BASE=${{ steps.vars.outputs.renku_base }}
Expand All @@ -333,7 +399,7 @@ jobs:
username: ${{ secrets.RENKU_DOCKER_USERNAME }}
password: ${{ secrets.RENKU_DOCKER_PASSWORD }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
uses: docker/setup-buildx-action@v3
- uses: actions/checkout@v3
- name: Set outputs
id: vars
Expand All @@ -351,7 +417,7 @@ jobs:
type=semver,pattern={{version}},prefix=
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }},prefix=
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
BASE_IMAGE=${{ env.DOCKER_PREFIX }}-vnc:${{ steps.vars.outputs.sha_short }}
Expand Down Expand Up @@ -388,7 +454,7 @@ jobs:
- name: Set up QEMU
uses: docker/[email protected]
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
uses: docker/setup-buildx-action@v3
- name: Set outputs
id: vars
run: |
Expand All @@ -405,7 +471,7 @@ jobs:
type=semver,pattern={{version}},prefix=${{ matrix.JULIA_VERSION }}-
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.JULIA_VERSION == '1.9.0' }}
- name: Build and load
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
RENKU_BASE=${{ steps.vars.outputs.renku_base }}
Expand All @@ -428,7 +494,7 @@ jobs:
working-directory: tests
command: npx mocha -r ts-node/register index.ts
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
RENKU_BASE=${{ steps.vars.outputs.renku_base }}
Expand Down Expand Up @@ -487,7 +553,7 @@ jobs:

- uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2.0.0
uses: docker/setup-buildx-action@v3
- name: Set outputs
id: vars
run: |
Expand All @@ -503,8 +569,13 @@ jobs:
type=sha,prefix=${{ matrix.RVERSION }}-
type=semver,pattern={{version}},prefix=${{ matrix.RVERSION }}-
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' && matrix.RVERSION == 'devel' }}
- name: Clean cache
run: |
df -h
rm -rf /opt/hostedtoolcache/*
df -h
- name: Build and load
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
RENKU_BASE=${{ steps.vars.outputs.renku_base }}
Expand All @@ -525,9 +596,10 @@ jobs:
TEST_SPEC: rstudio.cy.ts
with:
working-directory: tests
start: docker images
command: npx mocha -r ts-node/register index.ts
- name: Build and push
uses: docker/build-push-action@v4
uses: docker/build-push-action@v5
with:
build-args: |
RENKU_BASE=${{ steps.vars.outputs.renku_base }}
Expand Down
2 changes: 2 additions & 0 deletions docker/py/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,5 @@ COPY --chown=1000:100 --from=builder /opt/conda /opt/conda
COPY --chown=1000:100 --from=builder "$HOME/.renku" "$HOME/.renku"
RUN ln -s "$HOME/.renku/venv/bin/renku" "$HOME/.renku/bin/renku" && \
ln -s "$HOME/.renku/venv/bin/_toil_worker" "$HOME/.renku/bin/"
ARG CONDA_ENVS_DIRS=/opt/conda/envs
ENV CONDA_ENVS_DIRS=${CONDA_ENVS_DIRS}
Loading