♻️ Refactor: Model and Parameters (#1135)

♻️ Complete refactor of model and parameter packages using attrs (#1135) Summary of changes: * Added basic model. * Added megacomplexes model. * New Model can load from dict. * Basic idea for item fill. * Added get issues. * Added item fill. * Added model as_dict. * Added model markdown. * Added get parameter labels. * Added validate dataset datasetmodel megacomplexes. * Added global items. * Replaced old model with new. * Adapted optimization. * Adapted builtin megacomplexes. * Reimplemented ParameterGroup as Parameters. * Adapted yml load parameters. * All test are working again. * Fixed mypy issues and added docstrings for model package. * Fixed small bugs. * 🩹 Fixed benchmarks * ♻️ Refactored by Sourcery * Fixed project parameter generation. * 💥 Raised min python version to 3.10 * Refactored clp penalties to have types. * Deprecated parameter group. * Fixed model as_dict aand validate. * Fixed unique megacomplex check. * Fixed damped oscillation megacomplex type. * Small fixes. * 🩹🗑️ Raised deprecation version for new deprecations to 0.9.0 🧪🗑️ Added test for clp_area_penalties deprecation * 🧹 Renamed fuctions not conforming to snake case * ✨ Added equality check for Parameters * ♻️ Made nan_or_equal work with general values not only numeric and moved it to glotaran.utils.helpers * 👌 Added helper decorator to omit default values from repr in attrs class 👌 Made Parameter omit default values in repr * 👌 Added proper repr to Parameters * 🧪🩹 Fixed and reactivated pandas tests * 🩹📚 Changed read the docs config to work with python 3.10 * Fixed bug in interval item. * Fixed Parameters.copy. * Added test to check that initial parameters don't get overwritten. * Included review comments. * Updated changelog. * 🩹 Fixed integration benchmark * 👌Avoid redefining built-in type * 📚 Added change to changelog Co-authored-by: s-weigand <[email protected]> Co-authored-by: Sourcery AI <> Co-authored-by: Joris Snellenburg <[email protected]>
glotaran · Nov 4, 2022 · ac99524 · ac99524
1 parent c8d7722
commit ac99524
Show file tree

Hide file tree

Showing 127 changed files with 4,582 additions and 5,674 deletions.
diff --git a/.github/workflows/CI_CD_actions.yml b/.github/workflows/CI_CD_actions.yml
@@ -19,7 +19,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Run pre-commit
         uses: pre-commit/[email protected]
 
@@ -29,10 +29,10 @@ jobs:
     steps:
       - name: Check out repo
         uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install check manifest
         run: python -m pip install check-manifest 'setuptools>=62.4.0'
       - name: Run check manifest
@@ -47,10 +47,10 @@ jobs:
         with:
           conda-channels: conda-forge
           activate-conda: false
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           conda install -y pandoc
@@ -72,10 +72,10 @@ jobs:
         with:
           conda-channels: conda-forge
           activate-conda: false
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           conda install -y pandoc
@@ -96,10 +96,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           python -m pip install -U pip wheel
@@ -117,7 +117,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest, macOS-latest]
-        python-version: [3.8, 3.9, "3.10"]
+        python-version: ["3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -147,10 +147,10 @@ jobs:
     needs: [test, docs, docs-notebooks]
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: |
           python -m pip install -U pip wheel 'setuptools>=62.4.0'

diff --git a/.github/workflows/doxygen.yml b/.github/workflows/doxygen.yml
@@ -17,10 +17,10 @@ jobs:
     name: Doxygen callgraph
     runs-on: ubuntu-latest
     steps:
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install dependencies
         run: pip install requests
       - name: Get ref to check out

diff --git a/.github/workflows/full_benchmark.yml b/.github/workflows/full_benchmark.yml
@@ -15,10 +15,10 @@ jobs:
         with:
           fetch-depth: 0
           ref: main
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install ASV
         run: |
           pip install asv virtualenv

diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -32,10 +32,10 @@ jobs:
         example_name: ${{fromJson(needs.create-example-list.outputs.example-list)}}
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install pyglotaran
         run: |
           pip install wheel
@@ -93,10 +93,10 @@ jobs:
           echo "♻️ pyglotaran-examples commit: $(< comparison-results-current/example_commit_sha.txt)"
           echo "::endgroup::"
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
 
       - name: Install dependencies
         run: |

diff --git a/.github/workflows/pr_benchmark.yml b/.github/workflows/pr_benchmark.yml
@@ -12,10 +12,10 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: "3.10"
       - name: Install ASV
         run: |
           pip install 'asv!=0.5' virtualenv

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -22,6 +22,7 @@ repos:
         types: [file]
         types_or: [python, pyi]
         args: [--py38-plus]
+        exclude: "glotaran.model.item"
 
   - repo: https://github.com/MarcoGorelli/absolufy-imports
     rev: v0.3.1
@@ -105,17 +106,17 @@ repos:
         args:
           - "--select=D,DAR"
         name: "flake8 lint docstrings"
-        files: "^glotaran/(plugin_system|utils|deprecation|testing|optimization|parameter|project|simulation|model/property.py|builtin/io/pandas)"
+        files: "^glotaran/(plugin_system|utils|deprecation|testing|optimization|parameter|project|simulation|model|builtin/io/pandas)"
         exclude: "docs|tests?/"
         additional_dependencies: [flake8-docstrings, darglint==1.8.0]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.982
     hooks:
       - id: mypy
-        files: "^glotaran/(plugin_system|utils|deprecation|testing|optimization|parameter|project|simulation|model/property.py|builtin/io/pandas)"
+        files: "^glotaran/(plugin_system|utils|deprecation|testing|optimization|parameter|project|simulation|model|builtin/io/pandas)"
         exclude: "docs"
-        additional_dependencies: [types-all]
+        additional_dependencies: [types-all, types-attrs]
 
   - repo: https://github.com/econchick/interrogate
     rev: 1.5.0

diff --git a/.sourcery.yaml b/.sourcery.yaml
@@ -5,7 +5,7 @@ refactor:
     - simplify-boolean-comparison
     - simplify-len-comparison
     - remove-unnecessary-cast
-  python_version: "3.8"
+  python_version: "3.10"
 
 metrics:
   quality_threshold: 25.0

diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -131,7 +131,7 @@ Before you submit a pull request, check that it meets these guidelines:
 1.  The pull request should include tests.
 2.  If the pull request adds functionality, the docs should be updated. Put
     your new functionality into a function with a `docstring`_.
-3.  The pull request should work for Python 3.8 and 3.9
+3.  The pull request should work for Python 3.10
     Check your Github Actions ``https://github.com/<your_name_here>/pyglotaran/actions``
     and make sure that the tests pass for all supported Python versions.
 

diff --git a/README.md b/README.md
@@ -22,7 +22,7 @@ A common use case for the library is the analysis of time-resolved spectroscopy
 
 Prerequisites:
 
-- Python 3.8, 3.9 or 3.10
+- Python 3.10
 - On Windows only 64bit is supported
 
 Note for Windows Users: The easiest way to get python for Windows is via [Anaconda](https://www.anaconda.com/)

diff --git a/benchmark/benchmarks/integration/ex_two_datasets/benchmark.py b/benchmark/benchmarks/integration/ex_two_datasets/benchmark.py
@@ -1,5 +1,7 @@
 from pathlib import Path
 
+from glotaran import __version__
+
 try:
     # 0.4.0 -0.5.1
     from glotaran.analysis.optimize import optimize
@@ -28,15 +30,20 @@ class IntegrationTwoDatasets:
     def setup(self):
         dataset1 = load_dataset(SCRIPT_DIR / "data/data1.ascii")
         dataset2 = load_dataset(SCRIPT_DIR / "data/data2.ascii")
-        model = load_model(str(SCRIPT_DIR / "models/model.yml"))
         parameters = load_parameters(str(SCRIPT_DIR / "models/parameters.yml"))
+        addition_kwargs = {}
+        if int(__version__.split(".")[1]) < 7:
+            model = load_model(str(SCRIPT_DIR / "models/model_lt_0.7.0.yml"))
+            addition_kwargs["non_negative_least_squares"] = True
+        else:
+            model = load_model(str(SCRIPT_DIR / "models/model.yml"))
         self.scheme = Scheme(
             model,
             parameters,
             {"dataset1": dataset1, "dataset2": dataset2},
             maximum_number_function_evaluations=11,
-            non_negative_least_squares=True,
             optimization_method="TrustRegionReflection",
+            **addition_kwargs,
         )
 
     def time_optimize(self):

diff --git a/benchmark/benchmarks/integration/ex_two_datasets/models/model.yml b/benchmark/benchmarks/integration/ex_two_datasets/models/model.yml
@@ -1,4 +1,4 @@
-type: kinetic-spectrum
+default_megacomplex: decay
 
 dataset:
   dataset1:
@@ -33,16 +33,16 @@ initial_concentration:
 
 irf:
   irf1:
-    type: spectral-multi-gaussian
-    center: [irf.center]
-    width: [irf.width]
+    type: gaussian
+    center: irf.center
+    width: irf.width
   irf1_no_dispersion:
-    type: spectral-multi-gaussian
-    center: [irf.center]
-    width: [irf.width]
+    type: gaussian
+    center: irf.center
+    width: irf.width
 
 # It works without equal_area_penalties but then the inputs cannot be estimated
-equal_area_penalties:
+clp_penalties:
   - type: equal_area
     source: s1
     source_intervals: [[300, 3000]]
@@ -57,7 +57,6 @@ equal_area_penalties:
     target_intervals: [[300, 3000]]
     parameter: area.1
     weight: 0.1
-
 # Example of weight application:
 # weights:
 #   - datasets: [dataset1, dataset2]

diff --git a/benchmark/benchmarks/integration/ex_two_datasets/models/model_lt_0.7.0.yml b/benchmark/benchmarks/integration/ex_two_datasets/models/model_lt_0.7.0.yml
@@ -0,0 +1,65 @@
+type: kinetic-spectrum
+
+dataset:
+  dataset1:
+    megacomplex: [complex1]
+    initial_concentration: input1
+    irf: irf1
+    scale: scale.1
+  dataset2:
+    megacomplex: [complex1]
+    initial_concentration: input2
+    irf: irf1
+    scale: scale.2
+
+megacomplex:
+  complex1:
+    k_matrix: [km1]
+
+k_matrix:
+  km1:
+    matrix:
+      (s1, s1): "rates.k1"
+      (s2, s2): "rates.k2"
+      (s3, s3): "rates.k3"
+
+initial_concentration:
+  input1:
+    compartments: [s1, s2, s3]
+    parameters: [inputs.1, inputs.2, inputs.3]
+  input2:
+    compartments: [s1, s2, s3]
+    parameters: [inputs.1, inputs.7, inputs.8]
+
+irf:
+  irf1:
+    type: spectral-multi-gaussian
+    center: [irf.center]
+    width: [irf.width]
+  irf1_no_dispersion:
+    type: spectral-multi-gaussian
+    center: [irf.center]
+    width: [irf.width]
+
+# It works without equal_area_penalties but then the inputs cannot be estimated
+equal_area_penalties:
+  - type: equal_area
+    source: s1
+    source_intervals: [[300, 3000]]
+    target: s2
+    target_intervals: [[300, 3000]]
+    parameter: area.1
+    weight: 0.1
+  - type: equal_area
+    source: s1
+    source_intervals: [[300, 3000]]
+    target: s3
+    target_intervals: [[300, 3000]]
+    parameter: area.1
+    weight: 0.1
+# Example of weight application:
+# weights:
+#   - datasets: [dataset1, dataset2]
+#     global_interval: [100, 102]
+#     model_interval: [301, 502]
+#     value: 0.95
diff --git a/benchmark/pytest/analysis/test_optimization_group.py b/benchmark/pytest/analysis/test_optimization_group.py
@@ -10,7 +10,7 @@
 from glotaran.model import Model
 from glotaran.model import megacomplex
 from glotaran.optimization.optimization_group import OptimizationGroup
-from glotaran.parameter import ParameterGroup
+from glotaran.parameter import Parameters
 from glotaran.project import Scheme
 from glotaran.testing.plugin_system import monkeypatch_plugin_registry
 
@@ -32,11 +32,15 @@
     np.ones((TEST_AXIS_GLOBAL_SIZE, TEST_AXIS_MODEL_SIZE)),
     coords=(("global", TEST_AXIS_GLOBAL.data), ("test", TEST_AXIS_MODEL.data)),
 )
-TEST_PARAMETERS = ParameterGroup.from_list([])
+TEST_PARAMETERS = Parameters.from_list([])
 
 
-@megacomplex(dimension="test", properties={"is_index_dependent": bool})
+@megacomplex()
 class BenchmarkMegacomplex(Megacomplex):
+    dimension: str = "test"
+    type: str = "benchmark"
+    is_index_dependent: bool
+
     def calculate_matrix(
         self,
         dataset_model,
@@ -60,22 +64,21 @@ def finalize_data(
         pass
 
 
+BenchmarkModel = Model.create_class_from_megacomplexes([BenchmarkMegacomplex])
+
+
 @monkeypatch_plugin_registry(test_megacomplex={"benchmark": BenchmarkMegacomplex})
 def setup_model(index_dependent, link_clp):
     model_dict = {
-        "megacomplex": {"m1": {"is_index_dependent": index_dependent}},
+        "megacomplex": {"m1": {"type": "benchmark", "is_index_dependent": index_dependent}},
         "dataset_groups": {"default": {"link_clp": link_clp}},
         "dataset": {
             "dataset1": {"megacomplex": ["m1"]},
             "dataset2": {"megacomplex": ["m1"]},
             "dataset3": {"megacomplex": ["m1"]},
         },
     }
-    return Model.from_dict(
-        model_dict,
-        megacomplex_types={"benchmark": BenchmarkMegacomplex},
-        default_megacomplex_type="benchmark",
-    )
+    return BenchmarkModel(**model_dict)
 
 
 def setup_scheme(model):