Update DevOps tooling

Anaconda: - Remove unused `meta.yaml` (packages are now released through conda-forge feedstock) - Remove `meta.yaml` from `tbump.toml` - Rename `conda/` dir to `conda-env/` - Rename `e3sm_diags_env_dev.yml` to `dev.yml` and `e3sm_diags_env.yml` to `prod.yml` - Update dependencies to the latest versions - Update name and prefix of envs - Remove dev dependences in `prod.yml` - Add `ci.yml` for a minimal CI/CD conda env to speed up GH Actions - Only contains the minimum dependencies required to build the package and run the test suite - Add `docs.yml` for minimal CI/CD docs env to speed up GH Actions - Only contains the minimum dependencies required to build the documentation CI/CD: - Add Python matrix testing to `build_workflow.yml` - Use `ci.yml` instead of `dev.yml` when building conda env `setup.py`: - Update `setup()` contents with supported Python versions `setup.cfg` and `pre-commit-config.yaml` - Update dependencies to the latest versions Docs: - Update `docs/source/install.rst` with latest conda env updates
E3SM-Project · Feb 28, 2022 · ddd5908 · ddd5908
1 parent 04aee49
commit ddd5908
Show file tree

Hide file tree

Showing 25 changed files with 222 additions and 226 deletions.
diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml
@@ -9,80 +9,89 @@ on:
 
   workflow_dispatch:
 
+env:
+  CANCEL_OTHERS: true
+  PATHS_IGNORE: '["**/README.md", "**/docs/**", "**/examples/**", "**/misc/**", "**/.vscode/**"]'
+
 jobs:
-  check-jobs-to-skip:
+  pre-commit-hooks:
     runs-on: ubuntu-latest
-    outputs:
-      should_skip: ${{ steps.skip_check.outputs.should_skip }}
+    timeout-minutes: 2
     steps:
       - id: skip_check
         uses: fkirc/skip-duplicate-actions@master
         with:
-          cancel_others: true
-          paths_ignore: '["**/README.md", "**/docs/**", "**/examples/**", "**/misc/**"]'
+          cancel_others: ${{ env.CANCEL_OTHERS }}
+          paths_ignore: ${{ env.PATHS_IGNORE }}
 
-  pre-commit-hooks:
-    needs: check-jobs-to-skip
-    if: ${{ needs.check-jobs-to-skip.outputs.should_skip != 'true' }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 2
-    steps:
-      - name: Checkout Code Repository
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Checkout Code Repository
         uses: actions/checkout@v2
 
-      - name: Set up Python 3.7
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Set up Python 3.9
         uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.9
 
-      # Run all pre-commit hooks on all the files.
-      # Getting only staged files can be tricky in case a new PR is opened
-      # since the action is run on a branch in detached head state
-      - name: Install and Run Pre-commit
-        uses: pre-commit/[email protected]
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        # Run all pre-commit hooks on all the files.
+        # Getting only staged files can be tricky in case a new PR is opened
+        # since the action is run on a branch in detached head state
+        name: Install and Run Pre-commit
+        uses: pre-commit/[email protected]
 
   build:
-    needs: check-jobs-to-skip
-    if: ${{ needs.check-jobs-to-skip.outputs.should_skip != 'true' }}
-    runs-on: ubuntu-latest
+    name: Build (Python ${{ matrix.python-version }})
+    runs-on: "ubuntu-latest"
+    timeout-minutes: 20
     defaults:
       run:
         shell: bash -l {0}
-    timeout-minutes: 20
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9"]
     steps:
-      - uses: actions/checkout@v2
+      - id: skip_check
+        uses: fkirc/skip-duplicate-actions@master
+        with:
+          cancel_others: ${{ env.CANCEL_OTHERS }}
+          paths_ignore: ${{ env.PATHS_IGNORE }}
 
-      - name: Cache Conda
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        uses: actions/checkout@v2
+
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Cache Conda
         uses: actions/cache@v2
         env:
-          # Increase this value to reset cache if conda/e3sm_diags_env_dev.yml has not changed in the workflow
+          # Increase this value to reset cache if conda-env/ci.yml has not changed in the workflow
           CACHE_NUMBER: 0
         with:
           path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
-            hashFiles('conda/e3sm_diags_env_dev.yml') }}
+          key:
+            ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-${{
+            hashFiles('conda-env/dev.yml') }}
 
-      - name: Build Conda Environment
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Set up Conda Environment
         uses: conda-incubator/setup-miniconda@v2
         with:
-          activate-environment: e3sm_diags_env_dev
-          environment-file: conda/e3sm_diags_env_dev.yml
+          activate-environment: "e3sm_diags_ci"
+          environment-file: conda-env/ci.yml
           channel-priority: strict
           auto-update-conda: true
           # IMPORTANT: This needs to be set for caching to work properly!
           use-only-tar-bz2: true
+          python-version: ${{ matrix.python-version }}
 
-      - if: ${{ needs.check-jobs-to-skip.outputs.should_skip != 'true' }}
-        name: Show Conda Environment Info
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Install e3sm_diags
         run: |
-          conda config --set anaconda_upload no
-          conda info
-          conda list
-
-      - name: Install `e3sm-diags` Package
-        run: pip install .
+          python -m pip install .
 
-      - name: Run Tests
+      - if: ${{ steps.skip_check.outputs.should_skip != 'true' }}
+        name: Run Tests
         env:
           CHECK_IMAGES: True
         run: bash tests/test.sh
@@ -100,28 +109,25 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - name: Cache Conda
+        uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache if conda-env/docs.yml has not changed in the workflow
+          CACHE_NUMBER: 0
         with:
-          python-version: 3.7
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
+            hashFiles('conda-env/docs.yml') }}
 
-      - name: Cache pip
-        uses: actions/cache@v2
+      - name: Set up Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
         with:
-          # This path is specific to Ubuntu
-          path: ~/.cache/pip
-          # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-publish-docs
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
-
-      # Using pip for Sphinx dependencies because it takes too long to reproduce a conda environment (~10 secs vs. 3-4 mins)
-      # Low risk trade-off with mismatching dependencies in `e3sm_diags_env_dev.yml`
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install sphinx==4.0.2 sphinx_rtd_theme==0.5.2 sphinx-multiversion==0.2.4 docutils==0.16
+          activate-environment: "e3sm_diags_docs"
+          environment-file: conda-env/docs.yml
+          channel-priority: strict
+          auto-update-conda: true
+          # IMPORTANT: This needs to be set for caching to work properly!
+          use-only-tar-bz2: true
 
       - name: Build Sphinx Docs
         run: |

diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml
@@ -17,28 +17,25 @@ jobs:
           persist-credentials: false
           fetch-depth: 0
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - name: Cache pip
+      - name: Cache Conda
         uses: actions/cache@v2
+        env:
+          # Increase this value to reset cache if conda-env/docs.yml has not changed in the workflow
+          CACHE_NUMBER: 0
         with:
-          # This path is specific to Ubuntu
-          path: ~/.cache/pip
-          # Look to see if there is a cache hit for the corresponding requirements file
-          key: ${{ runner.os }}-pip-publish-docs
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
+          path: ~/conda_pkgs_dir
+          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
+            hashFiles('conda-env/docs.yml') }}
 
-      # Using pip for Sphinx dependencies because it takes too long to reproduce a conda environment (~10 secs vs. 3-4 mins)
-      # Low risk trade-off with mismatching dependencies in `e3sm_diags_env_dev.yml`
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install sphinx==4.0.2 sphinx_rtd_theme==0.5.2 sphinx-multiversion==0.2.4 docutils==0.16
+      - name: Set up Conda Environment
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          activate-environment: "e3sm_diags_docs"
+          environment-file: conda-env/docs.yml
+          channel-priority: strict
+          auto-update-conda: true
+          # IMPORTANT: This needs to be set for caching to work properly!
+          use-only-tar-bz2: true
 
       - name: Build Sphinx Docs
         run: |

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,33 +4,33 @@ fail_fast: true
 
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.1.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
       - id: check-yaml
 
   - repo: https://github.com/psf/black
-    rev: 21.5b2
+    rev: 22.1.0
     hooks:
       - id: black
 
   - repo: https://github.com/timothycrosley/isort
-    rev: 5.7.0
+    rev: 5.10.1
     hooks:
       - id: isort
 
   # Need to use flake8 GitHub mirror due to CentOS git issue with GitLab
   # https://github.com/pre-commit/pre-commit/issues/1206
   - repo: https://github.com/pycqa/flake8
-    rev: 3.9.2
+    rev: 4.0.1
     hooks:
       - id: flake8
         args: ["--config=setup.cfg"]
         additional_dependencies: [flake8-isort]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.812
+    rev: v0.931
     hooks:
       - id: mypy
         args: ["--config=setup.cfg"]
diff --git a/conda-env/ci.yml b/conda-env/ci.yml
@@ -0,0 +1,28 @@
+# Conda continuous integration/continuous deployment (CI/CD) for running the repository's GitHub Actions CI/CD workflows.
+# This env includes the latest dependencies required to build the package (similar to a conda recipe `meta.yaml` file).
+# End-users are not expected to use this environment.
+name: e3sm_diags_ci
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # ==================
+  - python>=3.8
+  - pip
+  - numpy
+  - matplotlib
+  - beautifulsoup4
+  - cartopy
+  - cartopy_offlinedata
+  - cdp
+  - cdms2
+  - cdtime
+  - cdutil
+  - genutil
+  - lxml
+  - netcdf4
+  # Required in test suite
+  - dask
+  - scipy
+prefix: /opt/miniconda3/envs/e3sm_diags_ci
diff --git a/conda-env/dev.yml b/conda-env/dev.yml
@@ -0,0 +1,47 @@
+# Conda development environment for testing local source code changes to `e3sm_diags` before merging them to production (`master` branch).
+name: e3sm_diags_dev
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # =================
+  - python=3.9.10
+  - pip=22.0.3
+  - numpy=1.22.2
+  - matplotlib=3.5.1
+  - beautifulsoup4=4.10.0
+  - cartopy=0.20.2
+  - cartopy_offlinedata=0.2.4
+  - cdp=1.7.0
+  - cdms2=3.1.5
+  - cdtime=3.1.4
+  - cdutil=8.2.1
+  - genutil=8.2.1
+  - lxml=4.8.0
+  - netcdf4=1.5.8
+  # Required in test suite
+  - dask=2022.2.0
+  - scipy=1.8.0
+  # Quality Assurance
+  # =================
+  # If versions are updated, also update 'rev' in `.pre-commit.config.yaml`
+  - black=22.1.0
+  - flake8=4.0.1
+  - flake8-isort=4.1.1
+  - isort=5.10.1
+  - mypy=0.931
+  - pre-commit=2.17.0
+  # Developer Tools
+  # =================
+  - tbump==6.7.0
+  # Documentation
+  # =================
+  - sphinx=4.3.2   # FIXME: > 4.3 conflicts with another package, maybe lxml (not sure)
+  - sphinx_rtd_theme=1.0.0
+  # Used when converting Jupyter notebooks to import to Sphinx
+  - nbconvert=6.4.2
+  - pandoc=2.17.1
+  - pip:
+      - sphinx-multiversion==0.2.4
+prefix: /opt/miniconda3/envs/e3sm_diags_dev
diff --git a/conda-env/docs.yml b/conda-env/docs.yml
@@ -0,0 +1,17 @@
+# Conda documentation environment for building documentation, including through automation in CI/CD.
+name: e3sm_diags_docs
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # =================
+  - python=3.9.10
+  - pip=22.0.3
+  # Documentation
+  # =================
+  - sphinx=4.4.0
+  - sphinx_rtd_theme=1.0.0
+  - pip:
+      - sphinx-multiversion==0.2.4
+prefix: /opt/miniconda3/envs/e3sm_diags_docs
diff --git a/conda-env/prod.yml b/conda-env/prod.yml
@@ -0,0 +1,12 @@
+# Conda production environment for production runs using the latest stable release of `e3sm_diags`.
+name: e3sm_diags_prod
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  # Base
+  # =================
+  - python=3.9.10
+  - pip=22.0.3
+  - e3sm_diags=2.6.1
+prefix: /opt/miniconda3/envs/e3sm_diags_prod
diff --git a/conda/e3sm_diags_env.yml b/conda/e3sm_diags_env.yml