Merge branch 'master' into yge/numpy

.github/workflows/benchmark.yml

@@ -1,12 +1,10 @@
 name: Benchmarks
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
       - master
-    paths-ignore:
-      - 'docs/**'
-      - 'devtools/**'
+    types: [opened, synchronize]
   workflow_dispatch:
     inputs:
       debug_enabled:
@@ -22,6 +20,12 @@ concurrency:
 jobs:
   benchmark:
     runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    strategy:
+      matrix:
+        python-version: ['3.9']
+        group: [1, 2]
 
     steps:
       # Enable tmate debugging of manually-triggered workflows if the input option was provided
@@ -31,41 +35,92 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: Set up Python 3.9
+
+      - name: Filter changes
+        id: changes
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            has_changes:
+              - 'desc/**'
+              - 'tests/benchmarks/**'
+              - 'requirements.txt'
+              - 'devtools/dev-requirements.txt'
+              - 'setup.cfg'
+              - '.github/workflows/benchmark.yml'
+
+      - name: Check for relevant changes
+        id: check_changes
+        run: echo "has_changes=${{ steps.changes.outputs.has_changes }}" >> $GITHUB_ENV
+
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
-      - name: Install dependencies
+          python-version: ${{ matrix.python-version }}
+
+      - name: Restore Python environment cache
+        if: env.has_changes == 'true'
+        id: restore-env
+        uses: actions/cache/restore@v4
+        with:
+          path: .venv-${{ matrix.python-version }}
+          key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }}
+
+      - name: Set up virtual environment if not restored from cache
+        if: steps.restore-env.outputs.cache-hit != 'true' && env.has_changes == 'true'
         run: |
+          gh cache list
+          python -m venv .venv-${{ matrix.python-version }}
+          source .venv-${{ matrix.python-version }}/bin/activate
           python -m pip install --upgrade pip
           pip install -r devtools/dev-requirements.txt
-      - name: Benchmark with pytest-benchmark
+          pip install matplotlib==3.9.2
+
+      - name: Benchmark with pytest-benchmark (PR)
+        if: env.has_changes == 'true'
         run: |
+          source .venv-${{ matrix.python-version }}/bin/activate
           pwd
           lscpu
           cd tests/benchmarks
           python -m pytest benchmark_cpu_small.py -vv \
             --benchmark-save='Latest_Commit' \
             --durations=0 \
-            --benchmark-save-data
+            --benchmark-save-data \
+            --splits 2 \
+            --group ${{ matrix.group }} \
+            --splitting-algorithm least_duration
+
       - name: Checkout current master
+        if: env.has_changes == 'true'
         uses: actions/checkout@v4
         with:
           ref: master
           clean: false
+
       - name: Checkout benchmarks from PR head
+        if: env.has_changes == 'true'
         run: git checkout ${{ github.event.pull_request.head.sha }} -- tests/benchmarks
-      - name: Benchmark with pytest-benchmark
+
+      - name: Benchmark with pytest-benchmark (MASTER)
+        if: env.has_changes == 'true'
         run: |
+          source .venv-${{ matrix.python-version }}/bin/activate
           pwd
           lscpu
           cd tests/benchmarks
           python -m pytest benchmark_cpu_small.py -vv \
             --benchmark-save='master' \
             --durations=0 \
-            --benchmark-save-data
-      - name: put benchmark results in same folder
+            --benchmark-save-data \
+            --splits 2 \
+            --group ${{ matrix.group }} \
+            --splitting-algorithm least_duration
+
+      - name: Put benchmark results in same folder
+        if: env.has_changes == 'true'
         run: |
+          source .venv-${{ matrix.python-version }}/bin/activate
           pwd
           cd tests/benchmarks
           find .benchmarks/ -type f -printf "%T@ %p\n" | sort -n | cut -d' ' -f 2- | tail -n 1 > temp1
@@ -75,22 +130,36 @@ jobs:
           mkdir compare_results
           cp $t1 compare_results
           cp $t2 compare_results
+
+      - name: Download artifact
+        if: always() && env.has_changes == 'true'
+        uses: actions/download-artifact@v4
+        with:
+          pattern: benchmark_artifact_*
+          path: tests/benchmarks
+
       - name: Compare latest commit results to the master branch results
+        if: env.has_changes == 'true'
         run: |
-          pwd
+          source .venv-${{ matrix.python-version }}/bin/activate
           cd tests/benchmarks
+          pwd
           python compare_bench_results.py
           cat commit_msg.txt
-      - name: comment PR with the results
+
+      - name: Comment PR with the results
+        if: env.has_changes == 'true'
         uses: thollander/actions-comment-pull-request@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           filePath: tests/benchmarks/commit_msg.txt
           comment_tag: benchmark
+
       - name: Upload benchmark data
-        if: always()
+        if: always() && env.has_changes == 'true'
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark_artifact
+          name: benchmark_artifact_${{ matrix.group }}
           path: tests/benchmarks/.benchmarks
+          include-hidden-files: true

.github/workflows/black.yml

@@ -6,26 +6,48 @@ jobs:
   black_format:
 
     runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
-      - name: Install dependencies
+          python-version: ${{ matrix.python-version }}
+
+      - name: Restore Python environment cache
+        id: restore-env
+        uses: actions/cache/restore@v4
+        with:
+          path: .venv-${{ matrix.python-version }}
+          key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }}
+
+      - name: Set up virtual environment if not restored from cache
+        if: steps.restore-env.outputs.cache-hit != 'true'
         run: |
+          gh cache list
+          python -m venv .venv-${{ matrix.python-version }}
+          source .venv-${{ matrix.python-version }}/bin/activate
           python -m pip install --upgrade pip
           pip install -r devtools/dev-requirements.txt
+
       - name: Check files using the black formatter
         run: |
+          source .venv-${{ matrix.python-version }}/bin/activate
           black --version
           black --check desc/ tests/ || black_return_code=$?
           echo "BLACK_RETURN_CODE=$black_return_code" >> $GITHUB_ENV
           black desc/ tests/
+
       - name: Annotate diff changes using reviewdog
         uses: reviewdog/action-suggester@v1
         with:
           tool_name: blackfmt
+
       - name: Fail if not formatted
         run: |
           exit ${{ env.BLACK_RETURN_CODE }}

.github/workflows/cache_dependencies.yml

@@ -0,0 +1,58 @@
+name: Cache dependencies
+# This workflow is triggered every 2 days and updates the Python
+# and pip dependencies cache
+on:
+  schedule:
+    - cron: '30 8 */2 * *'  # This triggers the workflow at 4:30 AM ET every 2 days
+    # cron syntax uses UTC time, so 4:30 AM ET is 8:30 AM UTC (for daylight time)
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      GH_TOKEN: ${{ github.token }}
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Delete old cached file with same python version
+        run: |
+          echo "Current Cached files list"
+          gh cache list
+          echo "Deleting cached files with pattern: ${{ runner.os }}-venv-${{ matrix.python-version }}-"
+          for cache_key in $(gh cache list --json key -q ".[] | select(.key | startswith(\"${{ runner.os }}-venv-${{ matrix.python-version }}-\")) | .key"); do
+            echo "Deleting cache with key: $cache_key"
+            gh cache delete "$cache_key"
+          done
+
+      # Update the matplotlib version if needed later
+      - name: Set up virtual environment
+        run: |
+          python -m venv .venv-${{ matrix.python-version }}
+          source .venv-${{ matrix.python-version }}/bin/activate
+          python -m pip install --upgrade pip
+          pip install -r devtools/dev-requirements.txt
+          pip install matplotlib==3.9.2
+
+      - name: Cache Python environment
+        id: cache-env
+        uses: actions/cache@v4
+        with:
+          path: .venv-${{ matrix.python-version }}
+          key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }}
+
+      - name: Verify virtual environment activation
+        run: |
+          source .venv-${{ matrix.python-version }}/bin/activate
+          python --version
+          pip --version
+          pip list

.github/workflows/changelog_update.yml

@@ -0,0 +1,35 @@
+name: Check changelog updated
+
+on:
+  pull_request:
+    branches:
+      - master
+    types: [opened, synchronize, labeled, unlabeled]
+
+jobs:
+  check_changelog_updated:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Filter changes
+        id: changes
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            has_changes:
+              - 'desc/**'
+              - 'requirements.txt'
+              - 'requirements_conda.yml'
+              - '.github/workflows/changelog_update.yml'
+
+      - name: Check for relevant changes
+        id: check_changes
+        run: echo "has_changes=${{ steps.changes.outputs.has_changes }}" >> $GITHUB_ENV
+
+      - uses: actions/checkout@v4
+
+      - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip_changelog') && env.has_changes == 'true'}}
+        uses: danieljimeneznz/[email protected]
+        with:
+          require-changes-to: |
+            CHANGELOG.md
+          token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/jax_tests.yml

@@ -12,34 +12,34 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        jax-version: [0.3.0,  0.3.1,  0.3.2,  0.3.3,  0.3.4,  0.3.5,
-                      0.3.6,  0.3.7,  0.3.8,  0.3.9,  0.3.10, 0.3.11,
-                      0.3.12, 0.3.13, 0.3.14, 0.3.15, 0.3.16, 0.3.17,
-                      0.3.19, 0.3.20, 0.3.21, 0.3.22, 0.3.23, 0.3.24,
-                      0.3.25, 0.4.1,  0.4.2,  0.4.3,  0.4.4,  0.4.5,
-                      0.4.6,  0.4.7,  0.4.8,  0.4.9,  0.4.10, 0.4.11,
-                      0.4.12, 0.4.13, 0.4.14, 0.4.16, 0.4.17, 0.4.18,
+        jax-version: [0.4.12, 0.4.13, 0.4.14, 0.4.16, 0.4.17,
                       0.4.18, 0.4.19, 0.4.20, 0.4.21, 0.4.22, 0.4.23,
-                      0.4.24, 0.4.25]
+                      0.4.24, 0.4.25, 0.4.26, 0.4.27, 0.4.28, 0.4.29,
+                      0.4.30, 0.4.31, 0.4.33, 0.4.34, 0.4.35]
+                      # 0.4.32 is not available on PyPI
+                      # earlier jax versions are not compatible with other
+                      # dependencies as of 2024-10-04
         group: [1, 2]
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.9
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
-          cache: pip
-      - name: Install dependencies
+          python-version: '3.10'
+      - name: Upgrade pip
         run: |
           python -m pip install --upgrade pip
-          pip install -r devtools/dev-requirements.txt
-          pip install matplotlib==3.5.0
-      - name: Remove jax
+      - name: Install dependencies with given JAX version
         run: |
-          pip uninstall jax jaxlib -y
-      - name: install jax
+          sed -i '/jax/d' ./requirements.txt
+          sed -i '1i\jax[cpu] == ${{ matrix.jax-version }}' ./requirements.txt
+          cat ./requirements.txt
+          pip install -r ./devtools/dev-requirements.txt
+      - name: Verify dependencies
         run: |
-          pip install "jax[cpu]==${{ matrix.jax-version }}"
+          python --version
+          pip --version
+          pip list
       - name: Test with pytest
         run: |
           pwd

.github/workflows/linting.yml

@@ -5,17 +5,24 @@ on: [pull_request, workflow_dispatch]
 jobs:
   flake8_linting:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
     name: Linting
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.10
+      - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: ${{ matrix.python-version }}
+
+      # For some reason, loading venv makes this way slower
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install -r devtools/dev-requirements.txt
+
       - name: flake8 Lint
         uses: reviewdog/action-flake8@v3
         with: