feat: multi-stage builds (#158)

R-minimal/.gitlab-ci.yml → R/.gitlab-ci.yml

@@ -2,6 +2,7 @@ variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
 
 stages:
   - build

R-minimal/Dockerfile → R/Dockerfile

@@ -1,15 +1,16 @@
 # For finding latest versions of the base image see
 # https://github.com/SwissDataScienceCenter/renkulab-docker
 ARG RENKU_BASE_IMAGE=renku/renkulab-r:4.2.0-0.13.1
-FROM ${RENKU_BASE_IMAGE}
 
 ########################################################
 #        Renku install section - do not edit           #
 
+FROM ${RENKU_BASE_IMAGE} as builder
+
 # RENKU_VERSION determines the version of the renku CLI
 # that will be used in this image. To find the latest version,
 # visit https://pypi.org/project/renku/#history.
-ARG RENKU_VERSION={{ __renku_version__ | default("2.0.0") }}
+ARG RENKU_VERSION={{ __renku_version__ | default("1.11.2") }}
 
 # Install renku from pypi or from github if a dev version
 RUN if [ -n "$RENKU_VERSION" ] ; then \
@@ -19,15 +20,17 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
             pip uninstall renku -y ; \
             gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(rc[[:digit:]]\+\)*\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\4/p") ; \
             if [ -n "$gitversion" ] ; then \
-                pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
+                pip install --no-cache-dir --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
             else \
-                pip install --force renku==${RENKU_VERSION} ;\
+                pip install --no-cache-dir --force renku==${RENKU_VERSION} ;\
             fi \
         fi \
     fi
-#             End renku install section                #
+#             End Renku install section                #
 ########################################################
 
+FROM ${RENKU_BASE_IMAGE}
+
 # Uncomment and adapt if code is to be included in the image
 # COPY src /code/src
 
@@ -48,4 +51,6 @@ RUN R -f /tmp/install.R
 
 # install the python dependencies
 COPY requirements.txt /tmp/
-RUN pip3 install -r /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt --no-cache-dir
+
+COPY --from=builder ${HOME}/.renku/venv ${HOME}/.renku/venv

julia-minimal/.gitlab-ci.yml → bioc/.gitlab-ci.yml

@@ -2,6 +2,7 @@ variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
 
 stages:
   - build

bioc-minimal/Dockerfile → bioc/Dockerfile

@@ -1,14 +1,16 @@
 # see https://github.com/SwissDataScienceCenter/renkulab-docker
 # to swap this image for the latest version available
-FROM renku/renkulab-bioc:RELEASE_3_15-0.13.1
+ARG RENKU_BASE_IMAGE=renku/renkulab-bioc:RELEASE_3_15-0.13.1
 
 ########################################################
 #        Renku install section - do not edit           #
 
+FROM ${RENKU_BASE_IMAGE} as builder
+
 # RENKU_VERSION determines the version of the renku CLI
 # that will be used in this image. To find the latest version,
 # visit https://pypi.org/project/renku/#history.
-ARG RENKU_VERSION={{ __renku_version__ | default("2.0.0") }}
+ARG RENKU_VERSION={{ __renku_version__ | default("1.11.2") }}
 
 # Install renku from pypi or from github if a dev version
 RUN if [ -n "$RENKU_VERSION" ] ; then \
@@ -24,9 +26,10 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
             fi \
         fi \
     fi
-#             End renku install section                #
+#             End Renku install section                #
 ########################################################
 
+FROM ${RENKU_BASE_IMAGE}
 
 # Uncomment and adapt if code is to be included in the image
 # COPY src /code/src
@@ -48,4 +51,6 @@ RUN R -f /tmp/install.R
 
 # install the python dependencies
 COPY requirements.txt /tmp/
-RUN pip3 install -r /tmp/requirements.txt
+RUN pip3 install -r /tmp/requirements.txt --no-cache-dir
+
+COPY --from=builder ${HOME}/.renku/venv ${HOME}/.renku/venv

bioc/{{ __sanitized_project_name__ }}.Rproj

@@ -0,0 +1,13 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX

bioc-minimal/.gitlab-ci.yml → julia/.gitlab-ci.yml

@@ -2,6 +2,7 @@ variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
 
 stages:
   - build

julia-minimal/Dockerfile → julia/Dockerfile

@@ -1,15 +1,16 @@
 # For finding latest versions of the base image see
 # https://github.com/SwissDataScienceCenter/renkulab-docker
 ARG RENKU_BASE_IMAGE=renku/renkulab-julia:1.7.1-0.13.1
-FROM ${RENKU_BASE_IMAGE}
 
 ########################################################
 #        Renku install section - do not edit           #
 
+FROM ${RENKU_BASE_IMAGE} as builder
+
 # RENKU_VERSION determines the version of the renku CLI
 # that will be used in this image. To find the latest version,
 # visit https://pypi.org/project/renku/#history.
-ARG RENKU_VERSION={{ __renku_version__ | default("2.0.0") }}
+ARG RENKU_VERSION={{ __renku_version__ | default("1.11.2") }}
 
 # Install renku from pypi or from github if a dev version
 RUN if [ -n "$RENKU_VERSION" ] ; then \
@@ -19,15 +20,17 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
             pip uninstall renku -y ; \
             gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(rc[[:digit:]]\+\)*\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\4/p") ; \
             if [ -n "$gitversion" ] ; then \
-                pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
+                pip install --no-cache-dir --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
             else \
-                pip install --force renku==${RENKU_VERSION} ;\
+                pip install --no-cache-dir --force renku==${RENKU_VERSION} ;\
             fi \
         fi \
     fi
 #             End Renku install section                #
 ########################################################
 
+FROM ${RENKU_BASE_IMAGE}
+
 # Uncomment and adapt if code is to be included in the image
 # COPY src /code/src
 
@@ -45,7 +48,7 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
 # install the python dependencies
 COPY requirements.txt environment.yml /tmp/
 RUN mamba env update -q -f /tmp/environment.yml && \
-    /opt/conda/bin/pip install -r /tmp/requirements.txt && \
+    /opt/conda/bin/pip install -r /tmp/requirements.txt --no-cache-dir && \
     mamba clean -y --all && \
     mamba env export -n "root"
 
@@ -54,3 +57,5 @@ COPY Project.toml Manifest.toml /tmp/
 RUN mkdir /tmp/julia-pkg && \
     cp /tmp/Project.toml /tmp/Manifest.toml /tmp/julia-pkg/ && \
     julia -e'using Pkg; Pkg.activate("/tmp/julia-pkg/"); Pkg.instantiate(); Pkg.precompile()'
+
+COPY --from=builder ${HOME}/.renku/venv ${HOME}/.renku/venv

manifest.yaml

@@ -1,19 +1,19 @@
-- folder: python-minimal
+- folder: python
   name: Basic Python (3.9) Project
   description: The simplest Python-3.9-based renku project with a basic directory structure and necessary supporting files.
   variables: {}
   icon: python-minimal.png
-- folder: R-minimal
+- folder: R
   name: Basic R (4.2.0) Project
   description: The simplest R-4.2.0-based renku project with a basic directory structure and necessary supporting files.
   variables: {}
   icon: R-minimal.png
-- folder: bioc-minimal
+- folder: bioc
   name: R-Bioconductor (3.15) Project
   description: The simplest R bioconductor-3.15-based renku project with a basic directory structure and necessary supporting files.
   variables: {}
   icon: bioconductor.png
-- folder: julia-minimal
+- folder: julia
   name: Basic Julia (1.7.1) Project
   description: The simplest Julia 1.7.1-based renku project with a basic directory structure and necessary supporting files.
   variables: {}

minimal/.gitlab-ci.yml

@@ -2,6 +2,7 @@ variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
 
 stages:
   - build

minimal/Dockerfile

@@ -1,15 +1,16 @@
 # For finding latest versions of the base image see
 # https://github.com/SwissDataScienceCenter/renkulab-docker
 ARG RENKU_BASE_IMAGE=renku/renkulab-py:3.9-0.13.1
-FROM ${RENKU_BASE_IMAGE}
 
 ########################################################
 #        Renku install section - do not edit           #
 
+FROM ${RENKU_BASE_IMAGE} as builder
+
 # RENKU_VERSION determines the version of the renku CLI
 # that will be used in this image. To find the latest version,
 # visit https://pypi.org/project/renku/#history.
-ARG RENKU_VERSION={{ __renku_version__ | default("2.0.0") }}
+ARG RENKU_VERSION={{ __renku_version__ | default("1.11.2") }}
 
 # Install renku from pypi or from github if a dev version
 RUN if [ -n "$RENKU_VERSION" ] ; then \
@@ -19,15 +20,17 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
             pip uninstall renku -y ; \
             gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(rc[[:digit:]]\+\)*\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\4/p") ; \
             if [ -n "$gitversion" ] ; then \
-                pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
+                pip install --no-cache-dir --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
             else \
-                pip install --force renku==${RENKU_VERSION} ;\
+                pip install --no-cache-dir --force renku==${RENKU_VERSION} ;\
             fi \
         fi \
     fi
-#             End renku install section                #
+#             End Renku install section                #
 ########################################################
 
+FROM ${RENKU_BASE_IMAGE}
+
 # Uncomment and adapt if code is to be included in the image
 # COPY src /code/src
 
@@ -53,3 +56,5 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
 #    /opt/conda/bin/pip install -r /tmp/requirements.txt && \
 #    conda clean -y --all && \
 #    conda env export -n "root"
+
+COPY --from=builder ${HOME}/.renku/venv ${HOME}/.renku/venv

python-minimal/.gitlab-ci.yml → python/.gitlab-ci.yml

@@ -2,6 +2,7 @@ variables:
   GIT_STRATEGY: fetch
   GIT_SSL_NO_VERIFY: "true"
   GIT_LFS_SKIP_SMUDGE: 1
+  DOCKER_BUILDKIT: 1
 
 stages:
   - build

python-minimal/Dockerfile → python/Dockerfile

@@ -1,16 +1,16 @@
 # For finding latest versions of the base image see
 # https://github.com/SwissDataScienceCenter/renkulab-docker
 ARG RENKU_BASE_IMAGE=renku/renkulab-py:3.9-0.13.1
-FROM ${RENKU_BASE_IMAGE}
-
 
 ########################################################
 #        Renku install section - do not edit           #
 
+FROM ${RENKU_BASE_IMAGE} as builder
+
 # RENKU_VERSION determines the version of the renku CLI
 # that will be used in this image. To find the latest version,
 # visit https://pypi.org/project/renku/#history.
-ARG RENKU_VERSION={{ __renku_version__ | default("2.0.0") }}
+ARG RENKU_VERSION={{ __renku_version__ | default("1.11.2") }}
 
 # Install renku from pypi or from github if a dev version
 RUN if [ -n "$RENKU_VERSION" ] ; then \
@@ -20,15 +20,17 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
             pip uninstall renku -y ; \
             gitversion=$(echo "$RENKU_VERSION" | sed -n "s/^[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+\(rc[[:digit:]]\+\)*\(\.dev[[:digit:]]\+\)*\(+g\([a-f0-9]\+\)\)*\(+dirty\)*$/\4/p") ; \
             if [ -n "$gitversion" ] ; then \
-                pip install --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
+                pip install --no-cache-dir --force "git+https://github.com/SwissDataScienceCenter/renku-python.git@$gitversion" ;\
             else \
-                pip install --force renku==${RENKU_VERSION} ;\
+                pip install --no-cache-dir --force renku==${RENKU_VERSION} ;\
             fi \
         fi \
     fi
-#             End renku install section                #
+#             End Renku install section                #
 ########################################################
 
+FROM ${RENKU_BASE_IMAGE}
+
 # Uncomment and adapt if code is to be included in the image
 # COPY src /code/src
 
@@ -46,6 +48,8 @@ RUN if [ -n "$RENKU_VERSION" ] ; then \
 # install the python dependencies
 COPY requirements.txt environment.yml /tmp/
 RUN mamba env update -q -f /tmp/environment.yml && \
-    /opt/conda/bin/pip install -r /tmp/requirements.txt && \
+    /opt/conda/bin/pip install -r /tmp/requirements.txt --no-cache-dir && \
     mamba clean -y --all && \
     mamba env export -n "root"
+
+COPY --from=builder ${HOME}/.renku/venv ${HOME}/.renku/venv