Merge branch 'main' into support/atomistic

.docker/aiida-core-base/Dockerfile

@@ -24,7 +24,7 @@ ENV SYSTEM_USER="${SYSTEM_USER}"
 
 # Install all OS dependencies for notebook server that starts but lacks all
 # features (e.g., download as all possible file formats)
-ENV DEBIAN_FRONTEND noninteractive
+ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update --yes && \
     # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as
     #   the ubuntu base image is rebuilt too seldom sometimes (less than once a month)
@@ -145,7 +145,7 @@ RUN set -x && \
 # Add ~/.local/bin to PATH where the dependencies get installed via pip
 # This require the package installed with `--user` flag in pip, which we set as default.
 ENV PATH=${PATH}:/home/${SYSTEM_USER}/.local/bin
-ENV PIP_USER 1
+ENV PIP_USER=1
 
 # Switch to root to install AiiDA and set AiiDA as service
 # Install AiiDA from source code

.github/CODEOWNERS

@@ -2,9 +2,7 @@
 # currently active dependency manager (DM) to trigger an automatic review
 # request from the DM upon changes. Please see AEP-002 for details:
 # https://github.com/aiidateam/AEP/tree/master/002_dependency_management
-setup.*                         @aiidateam/dependency-manager
-environment.yml                 @aiidateam/dependency-manager
-requirements*.txt               @aiidateam/dependency-manager
-pyproject.toml                  @aiidateam/dependency-manager
-utils/dependency_management.py  @aiidateam/dependency-manager
-.github/workflows/dm.yml        @aiidateam/dependency-manager
+environment.yml                 @unkcpz @agoscinski
+pyproject.toml                  @unkcpz @agoscinski
+uv.lock                         @unkcpz @agoscinski
+utils/dependency_management.py  @unkcpz @agoscinski

.github/actions/install-aiida-core/action.yml

@@ -12,8 +12,11 @@ inputs:
     required: false
   # NOTE: Hard-learned lesson: we cannot use type=boolean here, apparently :-(
   # https://stackoverflow.com/a/76294014
-  from-requirements:
-    description: Install aiida-core dependencies from pre-compiled requirements.txt file
+  # NOTE2: When installing from lockfile, aiida-core and its dependencies
+  # are installed in a virtual environment located in .venv directory.
+  # Subsuquent jobs steps must either activate the environment or use `uv run`
+  from-lock:
+    description: Install aiida-core dependencies from a uv lock file
     default: 'true'
     required: false
 
@@ -25,26 +28,20 @@ runs:
     with:
       python-version: ${{ inputs.python-version }}
 
-  - name: Install uv installer
-    run: curl --proto '=https' --tlsv1.2 -LsSf https://${{ env.UV_URL }} | sh
-    env:
-      UV_VERSION: 0.2.9
-      UV_URL: github.com/astral-sh/uv/releases/download/$UV_VERSION/uv-installer.sh
-    shell: bash
+  - name: Set up uv
+    uses: astral-sh/setup-uv@v4
+    with:
+      version: 0.5.6
 
-  - name: Install dependencies from requirements-py-*.txt
-    if: ${{ inputs.from-requirements == 'true' }}
-    run: uv pip install --system -r requirements/requirements-py-${{ inputs.python-version }}.txt
+  - name: Install dependencies from uv lock
+    if: ${{ inputs.from-lock == 'true' }}
+    # NOTE: We're asserting that the lockfile is up to date
+    # NOTE2: 'pre-commit' extra recursively contains other extras
+    # needed to run the tests.
+    run: uv sync --locked --extra pre-commit
     shell: bash
 
   - name: Install aiida-core
-    run: uv pip install --system ${{ env.NO_DEPS }} -e .${{ inputs.extras }}
-    env:
-        # Don't install dependencies if they were installed through requirements file AND
-        # if no extras are required.
-        #
-        # If this syntax looks weird to you, dear reader, know that this is
-        # GHA's way to do ternary operator. :-/
-        # https://docs.github.com/en/actions/learn-github-actions/expressions#example
-      NO_DEPS: ${{ (inputs.from-requirements == 'true' && inputs.extras == '')  && '--no-deps' || '' }}
+    if: ${{ inputs.from-lock != 'true' }}
+    run: uv pip install --system -e .${{ inputs.extras }}
     shell: bash

.github/system_tests/test_daemon.py

@@ -16,16 +16,6 @@
 import tempfile
 import time
 
-from aiida.common import StashMode, exceptions
-from aiida.engine import run, submit
-from aiida.engine.daemon.client import get_daemon_client
-from aiida.engine.persistence import ObjectLoader
-from aiida.engine.processes import CalcJob, Process
-from aiida.manage.caching import enable_caching
-from aiida.orm import CalcJobNode, Dict, Int, List, Str, load_code, load_node
-from aiida.orm.nodes.caching import NodeCaching
-from aiida.plugins import CalculationFactory, WorkflowFactory
-from aiida.workflows.arithmetic.add_multiply import add, add_multiply
 from workchains import (
     ArithmeticAddBaseWorkChain,
     CalcFunctionRunnerWorkChain,
@@ -39,6 +29,16 @@
     WorkFunctionRunnerWorkChain,
 )
 
+from aiida.common import StashMode, exceptions
+from aiida.engine import run, submit
+from aiida.engine.daemon.client import get_daemon_client
+from aiida.engine.persistence import ObjectLoader
+from aiida.engine.processes import CalcJob, Process
+from aiida.manage.caching import enable_caching
+from aiida.orm import CalcJobNode, Dict, Int, List, Str, load_code, load_node
+from aiida.orm.nodes.caching import NodeCaching
+from aiida.plugins import CalculationFactory, WorkflowFactory
+from aiida.workflows.arithmetic.add_multiply import add, add_multiply
 from tests.utils.memory import get_instances
 
 CODENAME_ADD = 'add@localhost'

.github/workflows/benchmark.yml

@@ -4,6 +4,9 @@ on:
   push:
     branches: [main]
     paths-ignore: [docs/**]
+  pull_request:
+    branches-ignore: [gh-pages]
+    paths: [.github/workflows/benchmark*]
 
 # https://docs.github.com/en/actions/using-jobs/using-concurrency
 concurrency:
@@ -54,9 +57,10 @@ jobs:
       uses: ./.github/actions/install-aiida-core
       with:
         python-version: '3.10'
+        from-lock: 'true'
 
     - name: Run benchmarks
-      run: pytest --benchmark-only --benchmark-json benchmark.json
+      run: uv run pytest --db-backend psql --benchmark-only --benchmark-json benchmark.json tests/
 
     - name: Store benchmark result
       uses: aiidateam/github-action-benchmark@v3
@@ -73,4 +77,4 @@ jobs:
         alert-threshold: 200%
         comment-on-alert: true
         fail-on-alert: false
-        alert-comment-cc-users: '@chrisjsewell,@giovannipizzi'
+        alert-comment-cc-users: '@giovannipizzi,@agoscinski,@GeigerJ2,@khsrali,@unkcpz'

.github/workflows/ci-code.yml

@@ -18,40 +18,8 @@ env:
 
 jobs:
 
-  check-requirements:
-
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-
-    steps:
-    - uses: actions/checkout@v4
-
-    - name: Set up Python 3.12
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-
-    - name: Install utils/ dependencies
-      run: pip install -r utils/requirements.txt
-
-    - name: Check requirements files
-      id: check_reqs
-      run: python ./utils/dependency_management.py check-requirements DEFAULT
-
-    - name: Create commit comment
-      if: failure() && steps.check_reqs.outputs.error
-      uses: peter-evans/commit-comment@v3
-      with:
-        path: pyproject.toml
-        body: |
-          ${{ steps.check_reqs.outputs.error }}
-
-          Click [here](https://github.com/aiidateam/aiida-core/wiki/AiiDA-Dependency-Management) for more information on dependency management.
-
   tests:
 
-    needs: [check-requirements]
-
     runs-on: ubuntu-latest
     timeout-minutes: 45
 
@@ -96,19 +64,20 @@ jobs:
         python-version: ${{ matrix.python-version }}
 
     - name: Setup environment
-      run: .github/workflows/setup.sh
+      # Note: The virtual environment in .venv was created by uv in previous step
+      run: source .venv/bin/activate && .github/workflows/setup.sh
 
     - name: Run test suite
       env:
         AIIDA_TEST_PROFILE: test_aiida
         AIIDA_WARN_v3: 1
       # Python 3.12 has a performance regression when running with code coverage
       # so run code coverage only for python 3.9.
-      run: pytest -v tests -m 'not nightly' ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
+      run: uv run pytest -n auto --db-backend psql -m 'not nightly' tests/ ${{ matrix.python-version == '3.9' && '--cov aiida' || '' }}
 
     - name: Upload coverage report
       if: matrix.python-version == 3.9 && github.repository == 'aiidateam/aiida-core'
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         name: aiida-pytests-py3.9
@@ -118,7 +87,6 @@ jobs:
 
   tests-presto:
 
-    needs: [check-requirements]
     runs-on: ubuntu-latest
     timeout-minutes: 20
 
@@ -139,12 +107,11 @@ jobs:
     - name: Run test suite
       env:
         AIIDA_WARN_v3: 0
-      run: pytest -m 'presto'
+      run: uv run pytest -n auto -m 'presto' tests/
 
 
   verdi:
 
-    needs: [check-requirements]
     runs-on: ubuntu-latest
     timeout-minutes: 10
 
@@ -155,7 +122,7 @@ jobs:
       uses: ./.github/actions/install-aiida-core
       with:
         python-version: '3.12'
-        from-requirements: 'false'
+        from-lock: 'false'
 
     - name: Run verdi tests
       run: |

.github/workflows/docs-build.yml

@@ -25,7 +25,7 @@ jobs:
       with:
         python-version: '3.9'
         extras: '[docs,tests,rest,atomic_tools]'
-        from-requirements: 'false'
+        from-lock: 'false'
 
     - name: Build HTML docs
       id: linkcheck

.github/workflows/nightly.yml

@@ -67,13 +67,15 @@ jobs:
       uses: ./.github/actions/install-aiida-core
       with:
         python-version: '3.11'
+        from-lock: 'true'
 
     - name: Setup environment
-      run: .github/workflows/setup.sh
+      # NOTE: virtual env in .venv created by uv in previous step
+      run: source .venv/bin/activate && .github/workflows/setup.sh
 
     - name: Run tests
       id: tests
-      run: .github/workflows/tests_nightly.sh
+      run: source .venv/bin/activate && .github/workflows/tests_nightly.sh
 
     - name: Slack notification
       # Always run this step (otherwise it would be skipped if any of the previous steps fail) but only if the
@@ -101,22 +103,9 @@ jobs:
       matrix:
         # Currently supported RMQ versions per:
         # https://www.rabbitmq.com/docs/which-erlang#compatibility-matrix
-        rabbitmq-version: ['3.11', '3.12', '3.13']
+        rabbitmq-version: ['3.11', '3.12', '3.13', '4.0']
 
     services:
-      postgres:
-        image: postgres:16
-        env:
-          POSTGRES_DB: test_aiida
-          POSTGRES_PASSWORD: ''
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-        - 5432:5432
       rabbitmq:
         image: rabbitmq:${{ matrix.rabbitmq-version }}-management
         ports:
@@ -131,21 +120,19 @@ jobs:
       uses: ./.github/actions/install-aiida-core
       with:
         python-version: '3.11'
-
-    - name: Install system dependencies
-      run: sudo apt update && sudo apt install postgresql
+        from-lock: 'true'
 
     - name: Setup SSH on localhost
-      run: .github/workflows/setup_ssh.sh
+      run: source .venv/bin/activate && .github/workflows/setup_ssh.sh
 
     - name: Suppress RabbitMQ version warning
-      run: verdi config set warnings.rabbitmq_version False
+      run: uv run verdi config set warnings.rabbitmq_version False
 
     - name: Run tests
       id: tests
       env:
         AIIDA_WARN_v3: 0
-      run: pytest -sv -m 'requires_rmq'
+      run: uv run pytest -s --db-backend sqlite -m 'requires_rmq' tests/
 
     - name: Slack notification
       # Always run this step (otherwise it would be skipped if any of the previous steps fail) but only if the

.github/workflows/release.yml

@@ -42,7 +42,7 @@ jobs:
       with:
         python-version: '3.11'
         extras: '[pre-commit]'
-        from-requirements: 'false'
+        from-lock: 'false'
 
     - name: Run pre-commit
       run: pre-commit run --all-files || ( git status --short ; git diff ; exit 1 )
@@ -54,19 +54,6 @@ jobs:
     timeout-minutes: 30
 
     services:
-      postgres:
-        image: postgres:10
-        env:
-          POSTGRES_DB: test_aiida
-          POSTGRES_PASSWORD: ''
-          POSTGRES_HOST_AUTH_METHOD: trust
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-        - 5432:5432
       rabbitmq:
         image: rabbitmq:3.8.14-management
         ports:
@@ -76,16 +63,11 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Install system dependencies
-      run: |
-        sudo apt update
-        sudo apt install postgresql graphviz
-
     - name: Install aiida-core
       uses: ./.github/actions/install-aiida-core
 
     - name: Run sub-set of test suite
-      run: pytest -sv -k 'requires_rmq'
+      run: pytest -s -m requires_rmq --db-backend=sqlite tests/
 
   publish: