New docker image based on pure ubuntu and gracefully handle services

.docker/README.md

@@ -0,0 +1,21 @@
+# AiiDA docker stacks
+
+### Build images locally
+
+To build the images, run `docker buildx bake -f build.json -f docker-bake.hcl --load` (tested with *docker buildx* version v0.8.2).
+
+The build system will attempt to detect the local architecture and automatically build images for it (tested with amd64 and arm64).
+You can also specify a custom platform with the `--platform`, example: `docker buildx bake -f build.json -f docker-bake.hcl --set *.platform=linux/amd64 --load`.
+
+### Test the build images locally
+
+Run
+
+```bash
+TAG=newly-baked python -m pytest -s tests
+```
+
+### Trigger a build on ghcr.io and dockerhub
+
+Only the PR open to the organization repository will trigger a build on ghcr.io.
+Push to dockerhub is triggered when making a release on github.

.docker/aiida-core-base/Dockerfile

@@ -0,0 +1,174 @@
+# syntax=docker/dockerfile:1
+
+# Inspired by jupyter's docker-stacks-fundation image:
+# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/Dockerfile
+
+ARG BASE=ubuntu:22.04
+
+FROM $BASE
+
+LABEL maintainer="AiiDA Team <[email protected]>"
+
+ARG SYSTEM_USER="aiida"
+ARG SYSTEM_UID="1000"
+ARG SYSTEM_GID="100"
+
+
+# Fix: https://github.com/hadolint/hadolint/wiki/DL4006
+# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+USER root
+
+ENV SYSTEM_USER="${SYSTEM_USER}"
+
+# Install all OS dependencies for notebook server that starts but lacks all
+# features (e.g., download as all possible file formats)
+ENV DEBIAN_FRONTEND noninteractive
+RUN apt-get update --yes && \
+    # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as
+    #   the ubuntu base image is rebuilt too seldom sometimes (less than once a month)
+    apt-get upgrade --yes && \
+    apt-get install --yes --no-install-recommends \
+    # - bzip2 is necessary to extract the micromamba executable.
+    bzip2 \
+    # - xz-utils is necessary to extract the s6-overlay.
+    xz-utils \
+    ca-certificates \
+    locales \
+    sudo \
+    # development tools
+    git \
+    openssh-client \
+    vim \
+    # the gcc compiler need to build some python packages e.g. psutil and pymatgen
+    build-essential \
+    wget && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
+    locale-gen
+
+# Install s6-overlay to handle startup and shutdown of services
+ARG S6_OVERLAY_VERSION=3.1.5.0
+RUN wget --progress=dot:giga -O /tmp/s6-overlay-noarch.tar.xz \
+        "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz" && \
+    tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
+    rm /tmp/s6-overlay-noarch.tar.xz
+
+RUN set -x && \
+    arch=$(uname -m) && \
+    wget --progress=dot:giga -O /tmp/s6-overlay-binary.tar.xz \
+        "https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-${arch}.tar.xz" && \
+    tar -C / -Jxpf /tmp/s6-overlay-binary.tar.xz && \
+    rm /tmp/s6-overlay-binary.tar.xz
+
+# Configure environment
+ENV CONDA_DIR=/opt/conda \
+    SHELL=/bin/bash \
+    SYSTEM_USER="${SYSTEM_USER}" \
+    SYSTEM_UID=${SYSTEM_UID} \
+    SYSTEM_GID=${SYSTEM_GID} \
+    LC_ALL=en_US.UTF-8 \
+    LANG=en_US.UTF-8 \
+    LANGUAGE=en_US.UTF-8
+ENV PATH="${CONDA_DIR}/bin:${PATH}" \
+    HOME="/home/${SYSTEM_USER}"
+
+
+# Copy a script that we will use to correct permissions after running certain commands
+COPY fix-permissions /usr/local/bin/fix-permissions
+RUN chmod a+rx /usr/local/bin/fix-permissions
+
+# Enable prompt color in the skeleton .bashrc before creating the default SYSTEM_USER
+# hadolint ignore=SC2016
+RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
+   # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
+   echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
+
+# Create SYSTEM_USER with name jovyan user with UID=1000 and in the 'users' group
+# and make sure these dirs are writable by the `users` group.
+RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
+    sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
+    sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
+    useradd -l -m -s /bin/bash -N -u "${SYSTEM_UID}" "${SYSTEM_USER}" && \
+    mkdir -p "${CONDA_DIR}" && \
+    chown "${SYSTEM_USER}:${SYSTEM_GID}" "${CONDA_DIR}" && \
+    chmod g+w /etc/passwd && \
+    fix-permissions "${HOME}" && \
+    fix-permissions "${CONDA_DIR}"
+
+USER ${SYSTEM_UID}
+
+# Pin python version here
+ARG PYTHON_VERSION
+
+# Download and install Micromamba, and initialize Conda prefix.
+#   <https://github.com/mamba-org/mamba#micromamba>
+#   Similar projects using Micromamba:
+#     - Micromamba-Docker: <https://github.com/mamba-org/micromamba-docker>
+#     - repo2docker: <https://github.com/jupyterhub/repo2docker>
+# Install Python, Mamba and jupyter_core
+# Cleanup temporary files and remove Micromamba
+# Correct permissions
+# Do all this in a single RUN command to avoid duplicating all of the
+# files across image layers when the permissions change
+COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" initial-condarc "${CONDA_DIR}/.condarc"
+WORKDIR /tmp
+RUN set -x && \
+    arch=$(uname -m) && \
+    if [ "${arch}" = "x86_64" ]; then \
+        # Should be simpler, see <https://github.com/mamba-org/mamba/issues/1437>
+        arch="64"; \
+    fi && \
+    wget --progress=dot:giga -O /tmp/micromamba.tar.bz2 \
+        "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \
+    tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \
+    rm /tmp/micromamba.tar.bz2 && \
+    PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \
+    if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \
+    # Install the packages
+    ./micromamba install \
+        --root-prefix="${CONDA_DIR}" \
+        --prefix="${CONDA_DIR}" \
+        --yes \
+        "${PYTHON_SPECIFIER}" \
+        'mamba' && \
+    rm micromamba && \
+    # Pin major.minor version of python
+    mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
+    mamba clean --all -f -y && \
+    fix-permissions "${CONDA_DIR}" && \
+    fix-permissions "/home/${SYSTEM_USER}"
+
+# Add ~/.local/bin to PATH where the dependencies get installed via pip
+# This require the package installed with `--user` flag in pip
+ENV PATH=${PATH}:/home/${NB_USER}/.local/bin
+
+# Switch to root to install AiiDA and set AiiDA as service
+# Install AiiDA from source code
+USER root
+COPY --from=src . /tmp/aiida-core
+RUN pip install /tmp/aiida-core --no-cache-dir && \
+    rm -rf /tmp/aiida-core
+
+# Enable verdi autocompletion.
+RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \
+     echo 'eval "$(_VERDI_COMPLETE=bash_source verdi)"' >> "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \
+     chmod +x "${CONDA_DIR}/etc/conda/activate.d/activate_aiida_autocompletion.sh" && \
+     fix-permissions "${CONDA_DIR}"
+
+# COPY AiiDA profile configuration for profile setup init script
+COPY --chown="${SYSTEM_UID}:${SYSTEM_GID}" s6-assets/config-quick-setup.yaml "/aiida/assets/config-quick-setup.yaml"
+COPY s6-assets/s6-rc.d /etc/s6-overlay/s6-rc.d
+COPY s6-assets/init /etc/init
+
+# Otherwise will stuck on oneshot services
+# https://github.com/just-containers/s6-overlay/issues/467
+ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0
+
+# Switch back to USER aiida to avoid accidental container runs as root
+USER ${SYSTEM_UID}
+
+ENTRYPOINT ["/init"]
+
+WORKDIR "${HOME}"

.docker/aiida-core-base/fix-permissions

@@ -0,0 +1,35 @@
+#!/bin/bash
+# This is brought from jupyter docker-stacks:
+# https://github.com/jupyter/docker-stacks/blob/main/docker-stacks-foundation/fix-permissions
+# set permissions on a directory
+# after any installation, if a directory needs to be (human) user-writable,
+# run this script on it.
+# It will make everything in the directory owned by the group ${SYSTEM_GID}
+# and writable by that group.
+
+# uses find to avoid touching files that already have the right permissions,
+# which would cause massive image explosion
+
+# right permissions are:
+# group=${SYSEM_GID}
+# AND permissions include group rwX (directory-execute)
+# AND directories have setuid,setgid bits set
+
+set -e
+
+for d in "$@"; do
+    find "${d}" \
+        ! \( \
+            -group "${SYSTEM_GID}" \
+            -a -perm -g+rwX \
+        \) \
+        -exec chgrp "${SYSTEM_GID}" -- {} \+ \
+        -exec chmod g+rwX -- {} \+
+    # setuid, setgid *on directories only*
+    find "${d}" \
+        \( \
+            -type d \
+            -a ! -perm -6000 \
+        \) \
+        -exec chmod +6000 -- {} \+
+done

.docker/aiida-core-base/initial-condarc

@@ -0,0 +1,6 @@
+# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html
+
+auto_update_conda: false
+show_channel_urls: true
+channels:
+  - conda-forge

.docker/aiida-core-base/s6-assets/config-quick-setup.yaml

@@ -0,0 +1,15 @@
+---
+db_engine: postgresql_psycopg2
+db_backend: core.psql_dos
+db_host: database
+db_port: 5432
+su_db_username: postgres
+su_db_password: password
+su_db_name: template1
+db_name: aiida_db
+db_username: aiida
+db_password: password
+broker_host: messaging
+broker_port: 5672
+broker_username: guest
+broker_password: guest

.docker/opt/configure-aiida.sh → .docker/aiida-core-base/s6-assets/init/aiida-prepare.sh

@@ -2,17 +2,20 @@
 
 # This script is executed whenever the docker container is (re)started.
 
-# Debugging.
-set -x
-
 # Environment.
 export SHELL=/bin/bash
 
-# Setup AiiDA autocompletion.
-grep _VERDI_COMPLETE /home/${SYSTEM_USER}/.bashrc &> /dev/null || echo 'eval "$(_VERDI_COMPLETE=source verdi)"' >> /home/${SYSTEM_USER}/.bashrc
+# Configure AiiDA.
+export SETUP_DEFAULT_AIIDA_PROFILE=true
+export AIIDA_PROFILE_NAME=default
+export AIIDA_USER_EMAIL=aiida@localhost
+export AIIDA_USER_FIRST_NAME=Giuseppe
+export AIIDA_USER_LAST_NAME=Verdi
+export AIIDA_USER_INSTITUTION=Khedivial
+export AIIDA_PROFILE_PATH=/aiida/assets/config-quick-setup.yaml
 
 # Check if user requested to set up AiiDA profile (and if it exists already)
-if [[ ${SETUP_DEFAULT_PROFILE} == true ]] && ! verdi profile show ${PROFILE_NAME} &> /dev/null; then
+if [[ ${SETUP_DEFAULT_AIIDA_PROFILE} == true ]] && ! verdi profile show ${AIIDA_PROFILE_NAME} &> /dev/null; then
     NEED_SETUP_PROFILE=true;
 else
     NEED_SETUP_PROFILE=false;
@@ -22,15 +25,23 @@ fi
 if [[ ${NEED_SETUP_PROFILE} == true ]]; then
 
     # Create AiiDA profile.
-    verdi quicksetup                           \
-        --non-interactive                      \
-        --profile "${PROFILE_NAME}"            \
-        --email "${USER_EMAIL}"                \
-        --first-name "${USER_FIRST_NAME}"      \
-        --last-name "${USER_LAST_NAME}"        \
-        --institution "${USER_INSTITUTION}"    \
-        --db-host "${DB_HOST:localhost}"    \
-        --broker-host "${BROKER_HOST:localhost}"
+    verdi quicksetup              \
+        --non-interactive                            \
+        --profile "${AIIDA_PROFILE_NAME}"            \
+        --email "${AIIDA_USER_EMAIL}"                \
+        --first-name "${AIIDA_USER_FIRST_NAME}"      \
+        --last-name "${AIIDA_USER_LAST_NAME}"        \
+        --institution "${AIIDA_USER_INSTITUTION}"    \
+        --config "${AIIDA_PROFILE_PATH}"
+
+    # Supress verdi version warning because we are using a development version
+    verdi config set warnings.development_version False
+
+    # Supress rabbitmq version warning
+    # If it is built using RMQ version > 3.8.15 (as we did for the `aiida-core` image) which has the issue as described in
+    # https://github.com/aiidateam/aiida-core/wiki/RabbitMQ-version-to-use
+    # We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf
+    verdi config set warnings.rabbitmq_version False
 
     # Setup and configure local computer.
     computer_name=localhost
@@ -52,39 +63,24 @@ if [[ ${NEED_SETUP_PROFILE} == true ]]; then
       exit 1
     fi
 
-    verdi computer show ${computer_name} || verdi computer setup   \
-        --non-interactive                                          \
-        --label "${computer_name}"                                 \
-        --description "this computer"                              \
-        --hostname "${computer_name}"                              \
+    verdi computer show ${computer_name} &> /dev/null || verdi computer setup \
+        --non-interactive                                               \
+        --label "${computer_name}"                                      \
+        --description "container computer"                                   \
+        --hostname "${computer_name}"                                   \
         --transport core.local                                          \
-        --scheduler core.direct                                    \
-        --work-dir /home/aiida/aiida_run/                          \
-        --mpirun-command "mpirun -np {tot_num_mpiprocs}"           \
-        --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} && \
-    verdi computer configure core.local "${computer_name}"              \
-        --non-interactive                                          \
+        --scheduler core.direct                                         \
+        --work-dir /home/${SYSTEM_USER}/aiida_run/                          \
+        --mpirun-command "mpirun -np {tot_num_mpiprocs}"                \
+        --mpiprocs-per-machine ${LOCALHOST_MPI_PROCS_PER_MACHINE} &&    \
+    verdi computer configure core.local "${computer_name}" \
+        --non-interactive                                               \
         --safe-interval 0.0
 fi
 
 
 # Show the default profile
 verdi profile show || echo "The default profile is not set."
 
-# Make sure that the daemon is not running, otherwise the migration will abort.
-verdi daemon stop
-
 # Migration will run for the default profile.
-verdi storage migrate --force || echo "Database migration failed."
-
-# Supress rabbitmq version warning for arm64 since
-# the it build using latest version rabbitmq from apt install
-# We explicitly set consumer_timeout to 100 hours in /etc/rabbitmq/rabbitmq.conf
-export ARCH=`uname -m`
-if [ "$ARCH" = "aarch64" ]; then \
-    verdi config set warnings.rabbitmq_version False
-fi
-
-
-# Daemon will start only if the database exists and is migrated to the latest version.
-verdi daemon start || echo "AiiDA daemon is not running."
+verdi storage migrate --force

.docker/aiida-core-base/s6-assets/s6-rc.d/aiida-daemon-start/down

@@ -0,0 +1 @@
+verdi daemon stop

.docker/aiida-core-base/s6-assets/s6-rc.d/aiida-daemon-start/timeout-up

@@ -0,0 +1 @@
+0

.docker/aiida-core-base/s6-assets/s6-rc.d/aiida-daemon-start/type

@@ -0,0 +1 @@
+oneshot

.docker/aiida-core-base/s6-assets/s6-rc.d/aiida-daemon-start/up

@@ -0,0 +1,3 @@
+#!/command/execlineb -S0
+
+verdi daemon start

.docker/aiida-core-base/s6-assets/s6-rc.d/aiida-prepare/timeout-up

@@ -0,0 +1 @@
+0

.docker/aiida-core-base/s6-assets/s6-rc.d/aiida-prepare/type

@@ -0,0 +1 @@
+oneshot

.docker/aiida-core-base/s6-assets/s6-rc.d/aiida-prepare/up

@@ -0,0 +1,4 @@
+#!/command/execlineb -S0
+
+foreground { s6-echo "Calling /etc/init/aiida-prepare" }
+/etc/init/aiida-prepare.sh