Devops: Refactor the GHA Docker build (#6396)

This should both increase speed and robustness of the build, main due to:

* Not uploading image as artifacts. (this does have a side-effect that the build
  can't happen from forks, maybe we can workaround that)
* Build multiplatform images together

Co-authored-by: Jusong Yu <[email protected]>

.docker/aiida-core-base/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
 
 # Inspired by jupyter's docker-stacks-fundation image:
-# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/Dockerfile
+# https://github.com/jupyter/docker-stacks/tree/main/images/docker-stacks-foundation/Dockerfile
 
 ARG BASE=ubuntu:22.04
 
@@ -87,7 +87,7 @@ RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashr
    # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
    echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
 
-# Create SYSTEM_USER with name jovyan user with UID=1000 and in the 'users' group
+# Create $SYSTEM_USER user with UID=1000 and 'users' group
 # and make sure these dirs are writable by the `users` group.
 RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
     sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
@@ -112,7 +112,7 @@ ARG MAMBA_VERSION
 #   Similar projects using Micromamba:
 #     - Micromamba-Docker: <https://github.com/mamba-org/micromamba-docker>
 #     - repo2docker: <https://github.com/jupyterhub/repo2docker>
-# Install Python, Mamba and jupyter_core
+# Install Python, Mamba
 # Cleanup temporary files and remove Micromamba
 # Correct permissions
 # Do all this in a single RUN command to avoid duplicating all of the

.docker/docker-bake.hcl

@@ -13,7 +13,7 @@ variable "ORGANIZATION" {
 }
 
 variable "REGISTRY" {
-  default = "docker.io/"
+  default = "ghcr.io/"
 }
 
 variable "PLATFORMS" {
@@ -27,7 +27,7 @@ variable "TARGETS" {
 function "tags" {
   params = [image]
   result = [
-    "${REGISTRY}${ORGANIZATION}/${image}:newly-baked"
+    "${REGISTRY}${ORGANIZATION}/${image}"
   ]
 }
 

.docker/docker-compose.aiida-core-base.yml

@@ -29,7 +29,7 @@ services:
       retries: 10
 
   aiida:
-    image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/aiida-core-base}:${TAG:-latest}
+    image: ${REGISTRY:-}${AIIDA_CORE_BASE_IMAGE:-aiidateam/aiida-core-base}${TAG:-}
     environment:
       RMQHOST: messaging
       TZ: Europe/Zurich

.docker/docker-compose.aiida-core-dev.yml

@@ -3,7 +3,7 @@ version: '3.4'
 services:
 
   aiida:
-    image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/aiida-core-dev}:${TAG:-latest}
+    image: ${REGISTRY:-}${AIIDA_CORE_DEV_IMAGE:-aiidateam/aiida-core-dev}${TAG:-}
     environment:
       TZ: Europe/Zurich
       SETUP_DEFAULT_AIIDA_PROFILE: 'true'

.docker/docker-compose.aiida-core-with-services.yml

@@ -3,7 +3,7 @@ version: '3.4'
 services:
 
   aiida:
-    image: ${REGISTRY:-}${BASE_IMAGE:-aiidateam/aiida-core-with-services}:${TAG:-latest}
+    image: ${REGISTRY:-}${AIIDA_CORE_WITH_SERVICES_IMAGE:-aiidateam/aiida-core-with-services}${TAG:-}
     environment:
       TZ: Europe/Zurich
       SETUP_DEFAULT_AIIDA_PROFILE: 'true'

.docker/pytest.ini

@@ -1,5 +1,5 @@
 [pytest]
 minversion = 7.0
-addopts = -ra -q
+addopts = -ra -q --strict-markers
 testpaths =
     tests

.docker/requirements.txt

@@ -1,8 +1,4 @@
-docker
-pre-commit
-pytest
-requests
-tabulate
-pytest-docker
-docker-compose
-pyyaml<=5.3.1
+docker~=7.0.0
+pytest~=8.2.0
+requests~=2.32.0
+pytest-docker~=3.1.0

.docker/tests/conftest.py

@@ -3,21 +3,33 @@
 
 import pytest
 
+TARGETS = ('aiida-core-base', 'aiida-core-with-services', 'aiida-core-dev')
 
-@pytest.fixture(
-    scope='session',
-    params=[
-        'aiida-core-base',
-        'aiida-core-with-services',
-        'aiida-core-dev',
-    ],
-)
-def variant(request):
-    return request.param
+
+def target_checker(value):
+    msg = f"Invalid image target '{value}', must be one of: {TARGETS}"
+    if value not in TARGETS:
+        raise pytest.UsageError(msg)
+    return value
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        '--variant',
+        action='store',
+        required=True,
+        help='target (image name) of the docker-compose file to use.',
+        type=target_checker,
+    )
+
+
+@pytest.fixture(scope='session')
+def variant(pytestconfig):
+    return pytestconfig.getoption('variant')
 
 
 @pytest.fixture(scope='session')
-def docker_compose_file(pytestconfig, variant):
+def docker_compose_file(variant):
     return f'docker-compose.{variant}.yml'
 
 

.github/actions/install-aiida-core/action.yml

@@ -28,7 +28,7 @@ runs:
   - name: Install uv installer
     run: curl --proto '=https' --tlsv1.2 -LsSf https://${{ env.UV_URL }} | sh
     env:
-      UV_VERSION: 0.1.35
+      UV_VERSION: 0.1.44
       UV_URL: github.com/astral-sh/uv/releases/download/$UV_VERSION/uv-installer.sh
     shell: bash
 

.github/workflows/docker-build.yml

@@ -0,0 +1,77 @@
+name: Build Docker images and upload them to ghcr.io
+
+env:
+  BUILDKIT_PROGRESS: plain
+
+on:
+  workflow_call:
+    inputs:
+      runsOn:
+        description: GitHub Actions Runner image
+        required: true
+        type: string
+      platforms:
+        description: Target platforms for the build (linux/amd64 and/or linux/arm64)
+        required: true
+        type: string
+    outputs:
+      images:
+        description: Images identified by digests
+        value: ${{ jobs.build.outputs.images }}
+
+jobs:
+  build:
+    name: ${{ inputs.platforms }}
+    runs-on: ${{ inputs.runsOn }}
+    timeout-minutes: 60
+    defaults:
+      run:
+        # Make sure we fail if any command in a piped command sequence fails
+        shell: bash -e -o pipefail {0}
+
+    outputs:
+      images: ${{ steps.bake_metadata.outputs.images }}
+
+    steps:
+
+    - name: Checkout Repo ⚡️
+      uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      if: ${{ inputs.platforms != 'linux/amd64' }}
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to GitHub Container Registry 🔑
+      uses: docker/login-action@v3
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Build and upload to ghcr.io 📤
+      id: build
+      uses: docker/bake-action@v4
+      with:
+        push: true
+        workdir: .docker/
+        # Using provenance to disable default attestation so it will build only desired images:
+        # https://github.com/orgs/community/discussions/45969
+        provenance: false
+        set: |
+          *.platform=${{ inputs.platforms }}
+          *.output=type=registry,push-by-digest=true,name-canonical=true
+          *.cache-to=type=gha,scope=${{ github.workflow }},mode=max
+          *.cache-from=type=gha,scope=${{ github.workflow }}
+        files: |
+          docker-bake.hcl
+          build.json
+
+    - name: Set output variables
+      id: bake_metadata
+      run: |
+        .github/workflows/extract-docker-image-names.sh | tee -a "${GITHUB_OUTPUT}"
+      env:
+        BAKE_METADATA: ${{ steps.build.outputs.metadata }}