cache pip dependencies in CI; pin setuptool[_scm] dependencies to fix…

… build; codecov action bump (#106) * cache pip dependencies in CI * h5py as fixed version * save cache * save at the end * print env * get path from purelib * use restore action: * show pytest * restore HDF5_LIBDIR variable * back with .[test] install * comment cache * comment restore cache * debug: restore cache after install * uncomment pip install * pytest without mpiexec * update path * debug: where does pytest binary is? * run with python -m * md5 as key, filter warning * cat build.log * proper warning message * setuptools < 69.2.0 * setup setuptools to 56 * rm build-backend * setuptools==69.1.0 * setuptools-scm==7.1.0 * remove hdf5 variables * codecov update and added badge
open-atmos · Apr 8, 2024 · fc9252c · fc9252c
1 parent 1ed8a39
commit fc9252c
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 16 deletions.
diff --git a/.github/workflows/tests+pypi.yml b/.github/workflows/tests+pypi.yml
@@ -85,8 +85,72 @@ jobs:
         run: |
           pylint --unsafe-load-any-extension=y --disable=fixme $(git ls-files '*.py')
 
+  tests_setup:
+    strategy:
+      matrix:
+        platform: [ubuntu-latest, macos-latest]
+        mpi: [ 'mpich', 'openmpi', 'intelmpi']
+        python-version: ["3.10"]
+        exclude:
+          # as of time of writing, mpi4py/setup-mpi does not support it
+          - platform: macos-latest
+            mpi: intelmpi
+
+          # issues with: *** The MPI_Comm_rank() function was called before MPI_INIT was invoked.
+          - platform: ubuntu-latest
+            mpi: intelmpi
+
+          # https://github.com/Homebrew/homebrew-core/issues/26974
+          - platform: macos-latest
+            mpi: mpich
+
+    runs-on: ${{ matrix.platform }}
+    steps:
+        - uses: actions/checkout@v2
+        - uses: actions/setup-python@v1
+          with:
+            python-version: ${{ matrix.python-version }}
+        - run: |
+            echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
+            echo toml_ci_md5=$(cat pyproject.toml .github/workflows/tests+pypi.yml \
+            | python -c "import hashlib;print(hashlib.md5(open('/dev/stdin','rb').read()).hexdigest())") >> $GITHUB_ENV
+        - id: cache
+          uses: actions/cache@v4
+          with:
+            path: ${{ env.pip_user_site }}
+            key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
+
+        - if: steps.cache.outputs.cache-hit != 'true'
+          uses: mpi4py/setup-mpi@v1
+          with:
+            mpi: ${{ matrix.mpi }}
+        - if: steps.cache.outputs.cache-hit != 'true' && matrix.mpi == 'mpich'
+          run: echo _ch="ch" >> $GITHUB_ENV
+        - if: steps.cache.outputs.cache-hit != 'true' &&  startsWith(matrix.platform, 'ubuntu-')
+          run: |
+            sudo apt-get update && sudo apt-get install -y libhdf5-mpi$_ch-dev pkg-config
+            lscpu
+        - if: steps.cache.outputs.cache-hit != 'true' && startsWith(matrix.platform, 'ubuntu-') && matrix.mpi == 'mpich'
+          run: |
+            echo HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich >> $GITHUB_ENV
+            echo HDF5_INCLUDEDIR=/usr/include/hdf5/mpich >> $GITHUB_ENV
+        - if: steps.cache.outputs.cache-hit != 'true' && startsWith(matrix.platform, 'macos-')
+          run: | 
+            brew install hdf5-mpi && echo HDF5_DIR=/opt/homebrew >> $GITHUB_ENV
+            sysctl -a | grep cpu | grep hw
+        - if: steps.cache.outputs.cache-hit != 'true'
+          run: |
+            HDF5_MPI="ON" CC=mpicc pip install --no-binary=h5py h5py==3.10.0
+            pip install -e .[tests]
+        - run: pip show numpy
+        - id: cache-save
+          uses: actions/cache/save@v4
+          with:
+            path: ${{ env.pip_user_site }}
+            key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
+
   tests:
-    needs: [zenodo_json, pylint, pdoc, precommit]
+    needs: [zenodo_json, pylint, pdoc, precommit, tests_setup]
     strategy:
       matrix:
         platform: [ubuntu-latest, macos-latest]
@@ -108,32 +172,32 @@ jobs:
             mpi: mpich
 
       fail-fast: false
+
     runs-on: ${{ matrix.platform }}
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
+      - run: |
+          echo pip_user_site=$(python -c "import sysconfig; print(sysconfig.get_path('purelib'))") >> $GITHUB_ENV
+          echo toml_ci_md5=$(cat pyproject.toml .github/workflows/tests+pypi.yml \
+          | python -c "import hashlib;print(hashlib.md5(open('/dev/stdin','rb').read()).hexdigest())") >> $GITHUB_ENV
       - uses: mpi4py/setup-mpi@v1
         with:
           mpi: ${{ matrix.mpi }}
+      - id: cache
+        uses: actions/cache/restore@v4
+        with:
+          path: ${{ env.pip_user_site }}
+          key: ${{ matrix.platform }}-${{ matrix.mpi }}-${{ matrix.python-version }}-${{ env.toml_ci_md5 }}
       - if: matrix.mpi == 'mpich'
         run: echo _ch="ch" >> $GITHUB_ENV
       - if: startsWith(matrix.platform, 'ubuntu-')
-        run: |
-          sudo apt-get update && sudo apt-get install -y libhdf5-mpi$_ch-dev pkg-config
-          lscpu
-      - if: startsWith(matrix.platform, 'ubuntu-') && matrix.mpi == 'mpich'
-        run: |
-          echo HDF5_LIBDIR=/usr/lib/x86_64-linux-gnu/hdf5/mpich >> $GITHUB_ENV
-          echo HDF5_INCLUDEDIR=/usr/include/hdf5/mpich >> $GITHUB_ENV
+        run: sudo apt-get update && sudo apt-get install -y libhdf5-mpi$_ch-dev pkg-config
       - if: startsWith(matrix.platform, 'macos-')
-        run: | 
-          brew install hdf5-mpi && echo HDF5_DIR=/opt/homebrew >> $GITHUB_ENV
-          sysctl -a | grep cpu | grep hw
-      - run: HDF5_MPI="ON" CC=mpicc pip install --no-binary=h5py h5py==3.10.0
-      - run: pip install -e .[tests]
+        run: brew install hdf5-mpi && echo HDF5_DIR=/opt/homebrew >> $GITHUB_ENV
       - run: python -We -c "import PyMPDATA_MPI"
       - if: matrix.mpi == 'openmpi'
         run: echo _mpiexec_args="--oversubscribe" >> $GITHUB_ENV
@@ -150,13 +214,13 @@ jobs:
             export COV_ARGS="--cov=./ --cov-report=xml"
             pip install pytest-cov
           fi
-          NUMBA_NUM_THREADS=3 mpiexec $_mpiexec_args -n ${{ matrix.mpi-np }} pytest $COV_ARGS --timeout=600 --timeout_method=thread -s -vv -We tests/local;
+          NUMBA_NUM_THREADS=3 mpiexec $_mpiexec_args -n ${{ matrix.mpi-np }} python -m pytest $COV_ARGS --timeout=600 --timeout_method=thread -s -vv -We tests/local;
       - uses: actions/upload-artifact@v2
         with:
           name: plots
           path: plots
       - if: env.CODECOV_RUN == '1'
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with: 
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true

diff --git a/README.md b/README.md
@@ -18,6 +18,8 @@
 [![Github Actions Build Status](https://github.com/open-atmos/PyMPDATA-MPI/workflows/tests+pypi/badge.svg?branch=main)](https://github.com/open-atmos/PyMPDATA-MPI/actions)
 [![PyPI version](https://badge.fury.io/py/PyMPDATA-MPI.svg)](https://pypi.org/project/PyMPDATA-MPI)
 [![API docs](https://img.shields.io/badge/API_docs-pdoc3-blue.svg)](https://open-atmos.github.io/PyMPDATA-MPI/)
+[![Coverage Status](https://codecov.io/gh/open-atmos/PyMPDATA-MPI/branch/main/graph/badge.svg)](https://app.codecov.io/gh/open-atmos/PyMPDATA-MPI)    
+
 
 PyMPDATA-MPI constitutes a [PyMPDATA](https://github.com/open-atmos/PyMPDATA) +
 [numba-mpi](https://github.com/numba-mpi/numba-mpi) coupler enabling numerical solutions

diff --git a/pyproject.toml b/pyproject.toml
@@ -4,7 +4,7 @@ py-modules = []
 [tool.setuptools_scm]
 
 [build-system]
-requires = ["setuptools", "setuptools-scm"]
+requires = ['setuptools==69.1.0', 'setuptools-scm==7.1.0']
 build-backend = "setuptools.build_meta"
 
 [project]