✨Backport of example notebooks

.github/requirements_ci.txt

@@ -4,3 +4,4 @@ jupyterlab>=3.0.0
 matplotlib>=3.3.0
 
 yaargh>=0.28.0
+papermill>=2.3.4

.github/workflows/integration_test.yml

@@ -49,7 +49,7 @@ jobs:
           # If not provided (push and pull_request event) it uses the default branch
           ref: ${{ github.event.inputs.pyglotaran_branch }}
       - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Install pyglotaran
@@ -68,21 +68,66 @@ jobs:
           examples_branch: ${{ github.event.inputs.pyglotaran_examples_branch }}
 
       - name: Upload Example Plots Artifact
-        uses: actions/upload-artifact@v3
+        if: always()
+        uses: actions/upload-artifact@v4
         with:
-          name: example-results
-          path: ${{ steps.example-run.outputs.plots-path }}
+          name: example-notebooks-${{ matrix.example_name }}
+          path: ${{ steps.example-run.outputs.notebook-path }}
 
       - name: Upload Example Results
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: example-results
+          name: example-results-${{ matrix.example_name }}
           path: ~/pyglotaran_examples_results
 
+  collect-artifacts:
+    if: always()
+    name: "Collect artifacts and reupload as bundel"
+    runs-on: ubuntu-latest
+    needs: [run-examples]
+    steps:
+      - name: Download Notebooks Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: example-notebooks
+          pattern: example-notebooks-*
+          merge-multiple: true
+
+      - name: Upload Example Notebooks Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: example-notebooks
+          path: example-notebooks
+          overwrite: true
+
+      - name: Delete Intermediate Notebooks artifacts
+        uses: GeekyEggo/delete-artifact@v5
+        with:
+          name: example-notebooks-*
+
+      - name: Download Result Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: example-results
+          pattern: example-results-*
+          merge-multiple: true
+
+      - name: Upload Example Result Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: example-results
+          path: example-results
+          overwrite: true
+
+      - name: Delete Intermediate Result artifacts
+        uses: GeekyEggo/delete-artifact@v5
+        with:
+          name: example-results-*
+
   compare-results:
     name: Compare Results
     runs-on: ubuntu-latest
-    needs: [run-examples]
+    needs: [collect-artifacts]
     steps:
       - name: Checkout compare results
         uses: actions/checkout@v4
@@ -92,7 +137,7 @@ jobs:
           path: comparison-results
 
       - name: Download result artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: example-results
           path: comparison-results-current
@@ -108,7 +153,7 @@ jobs:
           echo "::endgroup::"
 
       - name: Set up Python 3.10
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
 

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
       - id: check-ast
       - id: check-builtin-literals
@@ -10,33 +10,46 @@ repos:
       - id: debug-statements
       - id: fix-encoding-pragma
         args: [--remove]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.4.7
     hooks:
-      - id: pyupgrade
-        args: [--py38-plus]
-  - repo: https://github.com/psf/black
-    rev: 23.9.1
-    hooks:
-      - id: black
-        language_version: python3
-  - repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
-    hooks:
-      - id: isort
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
-    hooks:
-      - id: yesqa
-        additional_dependencies: [flake8-docstrings]
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
-    hooks:
-      - id: flake8
-        args: [--max-line-length=99]
+      - id: ruff
+        name: "ruff sort notebooks"
+        args:
+          - "--fix"
+          - "--select=I"
+          - "--config=.ruff-notebooks.toml"
+        alias: isort-notebooks
+        types_or: [jupyter]
+      - id: ruff-format
+        name: "ruff format notebooks"
+        types_or: [jupyter]
+        args:
+          - "--config=.ruff-notebooks.toml"
+        alias: format-notebooks
+      - id: ruff
+        name: "ruff lint notebooks"
+        types_or: [jupyter]
+        args:
+          - "--config=.ruff-notebooks.toml"
+        alias: check-notebooks
+
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         files: ".py|.rst"
-        args: ["--skip='*.ipynb'", "-L doas,DOAS"]
+        args: ["--ignore-words-list=doas"]
+        exclude: "\\.ipynb"
+
+  # Notebook tools
+  - repo: https://github.com/kynan/nbstripout
+    rev: 0.7.1
+    hooks:
+      - id: nbstripout
+        args:
+          - "--drop-empty-cells"
+          - "--extra-keys"
+          - "cell.metadata.tags metadata.vscode metadata.papermill metadata.language_info.version cell.metadata.papermill"

.ruff-notebooks.toml

@@ -0,0 +1,10 @@
+#:schema https://json.schemastore.org/ruff.json
+
+extend = ".ruff.toml"
+
+[lint]
+extend-ignore = ["D", "E402", "F404", "N811", "E703", "T201"]
+
+[lint.isort]
+required-imports = []
+force-single-line = false

.ruff.toml

@@ -0,0 +1,78 @@
+extend-exclude = ["venv", "docs/conf.py"]
+
+line-length = 99
+
+# Assume Python 3.10.
+target-version = "py310"
+
+# Enable using ruff with notebooks
+extend-include = ["*.ipynb"]
+
+[lint]
+select = [
+  "E", # pycodestyle
+  "W", # pycodestyle
+  # "C", # mccabe
+  "F", # pyflakes
+  # "UP", # pyupgrade
+  # "D", # pydocstyle
+  "N",   # pep8-naming
+  "YTT", # flake8-2020
+  "BLE", # flake8-blind-except
+  # "FBT", # flake8-boolean-trap
+  # "B", # flake8-bugbear
+  "C4",  # flake8-comprehensions
+  "T10", # flake8-debugger
+  "FA",  # flake8-future-annotations
+  # "EM", # flake8-errmsg
+  "ISC", # flake8-implicit-str-concat
+  "INP", # flake8-no-pep420
+  "PIE", # flake8-pie
+  "T20", # flake8-print
+  # "PT", # flake8-pytest-style
+  "RSE", # flake8-raise
+  "RET", # flake8-return
+  "SIM", # flake8-simplify
+  "TCH", # flake8-type-checking
+  # "ARG", # flake8-unused-arguments
+  "PTH", # flake8-use-pathlib
+  # "ERA", # eradicate
+  "PD",  # pandas-vet
+  "PGH", # pygrep-hooks
+  "NPY", # NumPy-specific
+  "RUF", # Ruff-specific
+]
+
+# Not actually external but we deactivated them for now
+external = ["D400", "D403", "D107", "D414"]
+
+ignore = [
+  "D202",
+  "D213",
+  "D203",
+  "ISC002",
+  "RUF001",
+  "RUF002",
+  "RUF003",
+  "B008",
+  # Covered by formatter
+  "ISC001",
+]
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+unfixable = ["F401"]
+
+[lint.per-file-ignores]
+"*/test_*.py" = ["ARG001", "RUF012", "N811", "T20", "PIE804"]
+"*/__init__.py" = ["F401"]
+"glotaran/builtin/io/netCDF/netCDF.py" = ["N999"]
+"docs/*" = ["INP001"]
+# Needs a full rewrite anyway
+"glotaran/builtin/io/ascii/wavelength_time_explicit_file.py" = ["PTH"]
+
+[lint.isort]
+required-imports = ["from __future__ import annotations"]
+known-first-party = ["glotaran"]
+force-single-line = true

action.yml

@@ -19,12 +19,12 @@ inputs:
     default: "false"
 
 outputs:
-  plots-path:
-    description: "Plot result path of example run"
-    value: ${{ steps.example-run.outputs.plots-path }}
   example-list:
     description: "List of all possible example names to be used in a workflow matrix."
     value: ${{ steps.example-run.outputs.example-list }}
+  notebook-path:
+    description: "Path to evaluated notebook after running the example"
+    value: ${{ steps.example-run.outputs.notebook-path }}
 
 runs:
   using: "composite"
@@ -62,17 +62,16 @@ runs:
         echo "::endgroup::"
       shell: bash
 
-    - name: Run example
+    - name: Run example notebooks
       id: example-run
       run: |
         echo "::group:: Running ${{ inputs.example_name }}"
         if [ '${{ inputs.set_example_list }}' = 'false' ]
         then
-          python pyglotaran-examples/scripts/run_examples.py ${{ inputs.example_name }} --headless 2>&1
-          echo "plots-path=pyglotaran-examples/plot_results"  >> $GITHUB_OUTPUT
+          python pyglotaran-examples/scripts/run_examples_notebooks.py ${{ inputs.example_name }} 2>&1
         else
-          pip install yaargh
-          python pyglotaran-examples/scripts/run_examples.py set-gha-example-list-output
+          pip install yaargh papermill
+          python pyglotaran-examples/scripts/run_examples_notebooks.py set-gha-example-list-output
         fi
 
         echo "::endgroup::"

pyglotaran_examples/.github/requirements_ci.txt

@@ -0,0 +1,7 @@
+
+pyglotaran>=0.3.0
+jupyterlab>=3.0.0
+matplotlib>=3.3.0
+
+yaargh>=0.28.0
+papermill>=2.3.4