❌ [TEST-PR] Wip/projects against d2b297

.github/CODEOWNERS

@@ -12,49 +12,55 @@
 # This should make it easy to add new rules without breaking existing ones.
 
 # Global rule:
-*                               @glotaran/admins
+*                                   @glotaran/admins
 
 # tooling
-/.github                        @glotaran/admins @glotaran/maintainers
-/.*                             @glotaran/admins @glotaran/maintainers
-/*.y*ml                         @glotaran/admins @glotaran/maintainers
-/*.ini                          @glotaran/admins @glotaran/maintainers
-/*.toml                         @glotaran/admins @glotaran/maintainers
-/*.txt                          @glotaran/admins @glotaran/maintainers
-LICENSE                         @glotaran/pyglotaran_creators
+/.github                            @glotaran/admins @glotaran/maintainers
+/.*                                 @glotaran/admins @glotaran/maintainers
+/*.y*ml                             @glotaran/admins @glotaran/maintainers
+/*.ini                              @glotaran/admins @glotaran/maintainers
+/*.toml                             @glotaran/admins @glotaran/maintainers
+/*.txt                              @glotaran/admins @glotaran/maintainers
+LICENSE                             @glotaran/pyglotaran_creators
 
 # docs
-/docs/**/*.rst                  @glotaran/maintainers @glotaran/pyglotaran_creators
-# /docs/**/*.md                 @glotaran/maintainers @glotaran/pyglotaran_creators
+/docs/**/*.rst                      @glotaran/maintainers @glotaran/pyglotaran_creators
+# /docs/**/*.md                     @glotaran/maintainers @glotaran/pyglotaran_creators
 
 # analysis module:
-/glotaran/analysis/             @jsnel @joernweissenborn
+/glotaran/analysis/                 @jsnel @joernweissenborn
 
 # builtin module:
-/glotaran/builtin/io/*          @glotaran/admins
-/glotaran/builtin/io/ascii      @jsnel @glotaran/maintainers
-/glotaran/builtin/io/csv        @glotaran/maintainers
-/glotaran/builtin/io/netCDF     @glotaran/maintainers
-/glotaran/builtin/io/sdt        @glotaran/maintainers
-/glotaran/builtin/models/       @jsnel @joernweissenborn
+/glotaran/builtin/io/*              @glotaran/admins
+/glotaran/builtin/io/ascii          @jsnel @glotaran/maintainers
+/glotaran/builtin/io/csv            @glotaran/maintainers
+/glotaran/builtin/io/netCDF         @glotaran/maintainers
+/glotaran/builtin/io/sdt            @glotaran/maintainers
+/glotaran/builtin/megacomplexes/    @jsnel @joernweissenborn
 
 # cli
-/glotaran/cli/                  @jsnel @glotaran/admins
+/glotaran/cli/                      @jsnel @glotaran/admins
 
 # examples
-/glotaran/examples/             @jsnel @glotaran/maintainers
+/glotaran/examples/                 @jsnel @glotaran/maintainers
 
 # io
-/glotaran/io/                   @jsnel @glotaran/maintainers
+/glotaran/io/                       @jsnel @glotaran/maintainers
 
 # model
-/glotaran/model/                @jsnel @glotaran/admins @joernweissenborn
+/glotaran/model/                    @jsnel @glotaran/admins @joernweissenborn
 
 # parameter
-/glotaran/parameter/            @jsnel @glotaran/admins @joernweissenborn
+/glotaran/parameter/                @jsnel @glotaran/admins @joernweissenborn
 
 # plugin_system
-glotaran/plugin_system          @s-weigand @glotaran/admins
+glotaran/plugin_system              @s-weigand @glotaran/admins
+
+# deprecation framework and tests
+glotaran/deprecation                @s-weigand @glotaran/admins
+
+# utility function
+glotaran/utils                      @s-weigand @glotaran/admins
 
 # project
-/glotaran/project/              @jsnel @glotaran/admins
+/glotaran/project/                  @jsnel @glotaran/admins

.gitignore

@@ -170,3 +170,4 @@ _summary.ps
 
 # benchmark results
 benchmark/.asv
+.benchmarks/

.pre-commit-config.yaml

@@ -57,34 +57,48 @@ repos:
         args: [--strip-empty-cells]
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 0.13.0
+    rev: 0.13.1
     hooks:
       - id: nbqa-black
-        additional_dependencies: [black==20.8b1]
+        additional_dependencies: [black==21.6b0]
         args: [--nbqa-mutate]
       - id: nbqa-pyupgrade
-        additional_dependencies: [pyupgrade==2.9.0]
+        additional_dependencies: [pyupgrade==2.19.4]
         args: [--nbqa-mutate, --py38-plus]
       - id: nbqa-flake8
       - id: nbqa-check-ast
       - id: nbqa-isort
-        additional_dependencies: [isort==5.7.0]
+        additional_dependencies: [isort==5.9.1]
         args: [--nbqa-mutate]
 
   # Linters
 
-  # - repo: https://github.com/PyCQA/pydocstyle
-  #   rev: 5.1.1
-  #   hooks:
-  #     - id: pydocstyle
-  #       exclude: "docs|tests"
-  #       # this is needed due to the following issue:
-  #       # https://github.com/PyCQA/pydocstyle/issues/368
-  #       args: [--ignore-decorators=wrap_func_as_method]
-  # - repo: https://github.com/terrencepreilly/darglint
-  #   rev: v1.5.5
-  #   hooks:
-  #     - id: darglint
+  - repo: https://github.com/PyCQA/pydocstyle
+    rev: 6.1.1
+    hooks:
+      - id: pydocstyle
+        files: "^glotaran/(plugin_system|utils|deprecation|project)"
+        exclude: "docs|tests?"
+        # this is needed due to the following issue:
+        # https://github.com/PyCQA/pydocstyle/issues/368
+        args: [--ignore-decorators=wrap_func_as_method]
+
+  - repo: https://github.com/terrencepreilly/darglint
+    rev: v1.8.0
+    hooks:
+      - id: darglint
+        files: "^glotaran/(plugin_system|utils|deprecation|project)"
+        exclude: "docs|tests?"
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.910
+    hooks:
+      - id: mypy
+        files: "^glotaran/(plugin_system|utils|deprecation|project)"
+        exclude: "docs"
+        args: [glotaran]
+        pass_filenames: false
+        additional_dependencies: [types-all]
 
   - repo: https://github.com/econchick/interrogate
     rev: 1.4.0
@@ -99,7 +113,7 @@ repos:
       - id: yesqa
         types: [file]
         types_or: [python, pyi]
-        additional_dependencies: [flake8-docstrings]
+        additional_dependencies: [flake8-docstrings, flake8-print]
 
   - repo: https://github.com/PyCQA/flake8
     rev: 3.9.2

CONTRIBUTING.rst

@@ -175,12 +175,15 @@ To make deprecations as robust as possible and give users all needed information
 to adjust their code, we provide helper functions inside the module
 :mod:`glotaran.deprecation`.
 
+.. currentmodule:: glotaran.deprecation.deprecation_utils
+
 The functions you most likely want to use are
 
 *   :func:`deprecate` for functions, methods and classes
 *   :func:`warn_deprecated` for call arguments
 *   :func:`deprecate_module_attribute` for module attributes
 *   :func:`deprecate_submodule` for modules
+*   :func:`deprecate_dict_entry` for dict entries
 
 
 Those functions not only make it easier to deprecate something, but they also check that
@@ -193,7 +196,7 @@ provides the test helper functions ``deprecation_warning_on_call_test_helper`` a
 Since the tests for deprecation are mainly for maintainability and not to test the
 functionality (those tests should be in the appropriate place)
 ``deprecation_warning_on_call_test_helper`` will by default just test that a
-``DeprecationWarning`` was raised and ignore all raise ``Exception`` s.
+``GlotaranApiDeprecationWarning`` was raised and ignore all raise ``Exception`` s.
 An exception to this rule is when adding back removed functionality
 (which shouldn't happen in the first place but might), which should be
 implemented in a file under ``glotaran/deprecation/modules`` and filenames should be like the
@@ -300,6 +303,17 @@ as an attribute to the parent package.
         to_be_removed_in_version="0.6.0",
     )
 
+Deprecating dict entries
+~~~~~~~~~~~~~~~~~~~~~~~~
+The possible dict deprecation actions are:
+
+- Swapping of keys ``{"foo": 1} -> {"bar": 1}`` (done via ``swap_keys=("foo", "bar")``)
+- Replacing of matching values ``{"foo": 1} -> {"foo": 2}`` (done via ``replace_rules=({"foo": 1}, {"foo": 2})``)
+- Replacing of matching values and swapping of keys ``{"foo": 1} -> {"bar": 2}`` (done via ``replace_rules=({"foo": 1}, {"bar": 2})``)
+
+For full examples have a look at the examples from the docstring (:func:`deprecate_dict_entry`).
+
+
 Deploying
 ---------
 

benchmark/benchmarks/integration/ex_two_datasets/benchmark.py

@@ -71,3 +71,9 @@ def peakmem_create_result(self):
         _create_result(
             self.problem, self.ls_result, self.free_parameter_labels, self.termination_reason
         )
+
+
+if __name__ == "__main__":
+    test = IntegrationTwoDatasets()
+    test.setup()
+    test.time_optimize()

benchmark/pytest/analysis/test_problem.py

@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pytest
+import xarray as xr
+
+from glotaran.analysis.problem_grouped import GroupedProblem
+from glotaran.analysis.problem_ungrouped import UngroupedProblem
+from glotaran.model import Megacomplex
+from glotaran.model import Model
+from glotaran.model import megacomplex
+from glotaran.parameter import ParameterGroup
+from glotaran.project import Scheme
+
+if TYPE_CHECKING:
+    from glotaran.model import DatasetModel
+
+TEST_AXIS_MODEL_SIZE = 100
+TEST_AXIS_MODEL = xr.DataArray(np.arange(0, TEST_AXIS_MODEL_SIZE))
+TEST_AXIS_GLOBAL_SIZE = 100
+TEST_AXIS_GLOBAL = xr.DataArray(np.arange(0, TEST_AXIS_GLOBAL_SIZE))
+TEST_CLP_SIZE = 20
+TEST_CLP_LABELS = [f"{i+1}" for i in range(TEST_CLP_SIZE)]
+TEST_MATRIX = np.ones((TEST_AXIS_MODEL_SIZE, TEST_CLP_SIZE))
+#  TEST_MATRIX = xr.DataArray(
+#      np.ones((TEST_AXIS_MODEL_SIZE, TEST_CLP_SIZE)),
+#      coords=(("test", TEST_AXIS_MODEL.data), ("clp_label", TEST_CLP_LABELS)),
+#  )
+TEST_DATA = xr.DataArray(
+    np.ones((TEST_AXIS_GLOBAL_SIZE, TEST_AXIS_MODEL_SIZE)),
+    coords=(("global", TEST_AXIS_GLOBAL.data), ("test", TEST_AXIS_MODEL.data)),
+)
+TEST_PARAMETER = ParameterGroup.from_list([])
+
+
+@megacomplex(dimension="test", properties={"is_index_dependent": bool})
+class BenchmarkMegacomplex(Megacomplex):
+    def calculate_matrix(self, dataset_model, indices, **kwargs):
+        return TEST_CLP_LABELS, TEST_MATRIX
+
+    def index_dependent(self, dataset_model):
+        return self.is_index_dependent
+
+    def finalize_data(
+        self,
+        dataset_model: DatasetModel,
+        dataset: xr.Dataset,
+        is_full_model: bool = False,
+        as_global: bool = False,
+    ):
+        pass
+
+
+def setup_model(index_dependent):
+    model_dict = {
+        "megacomplex": {"m1": {"is_index_dependent": index_dependent}},
+        "dataset": {
+            "dataset1": {"megacomplex": ["m1"]},
+            "dataset2": {"megacomplex": ["m1"]},
+            "dataset3": {"megacomplex": ["m1"]},
+        },
+    }
+    return Model.from_dict(
+        model_dict,
+        megacomplex_types={"benchmark": BenchmarkMegacomplex},
+        default_megacomplex_type="benchmark",
+    )
+
+
+def setup_scheme(model):
+    return Scheme(
+        model=model,
+        parameters=TEST_PARAMETER,
+        data={
+            "dataset1": TEST_DATA,
+            "dataset2": TEST_DATA,
+            "dataset3": TEST_DATA,
+        },
+    )
+
+
+def setup_problem(scheme, grouped):
+    return GroupedProblem(scheme) if grouped else UngroupedProblem(scheme)
+
+
+def test_benchmark_bag_creation(benchmark):
+
+    model = setup_model(False)
+    assert model.valid()
+
+    scheme = setup_scheme(model)
+    problem = setup_problem(scheme, True)
+
+    benchmark(problem.init_bag)
+
+
+@pytest.mark.parametrize("grouped", [True, False])
+@pytest.mark.parametrize("index_dependent", [True, False])
+def test_benchmark_calculate_matrix(benchmark, grouped, index_dependent):
+
+    model = setup_model(index_dependent)
+    assert model.valid()
+
+    scheme = setup_scheme(model)
+    problem = setup_problem(scheme, grouped)
+
+    if grouped:
+        problem.init_bag()
+
+    benchmark(problem.calculate_matrices)
+
+
+@pytest.mark.parametrize("grouped", [True, False])
+@pytest.mark.parametrize("index_dependent", [True, False])
+def test_benchmark_calculate_residual(benchmark, grouped, index_dependent):
+
+    model = setup_model(index_dependent)
+    assert model.valid()
+
+    scheme = setup_scheme(model)
+    problem = setup_problem(scheme, grouped)
+
+    if grouped:
+        problem.init_bag()
+    problem.calculate_matrices()
+
+    benchmark(problem.calculate_residual)
+
+
+@pytest.mark.parametrize("grouped", [True, False])
+@pytest.mark.parametrize("index_dependent", [True, False])
+def test_benchmark_calculate_result_data(benchmark, grouped, index_dependent):
+
+    model = setup_model(index_dependent)
+    assert model.valid()
+
+    scheme = setup_scheme(model)
+    problem = setup_problem(scheme, grouped)
+
+    if grouped:
+        problem.init_bag()
+    problem.calculate_matrices()
+    problem.calculate_residual()
+
+    benchmark(problem.create_result_data)
+
+
+#  @pytest.mark.skip(reason="To time consuming atm.")
+@pytest.mark.parametrize("grouped", [True, False])
+@pytest.mark.parametrize("index_dependent", [True, False])
+def test_benchmark_optimize_20_runs(benchmark, grouped, index_dependent):
+
+    model = setup_model(index_dependent)
+    assert model.valid()
+
+    scheme = setup_scheme(model)
+    problem = setup_problem(scheme, grouped)
+
+    @benchmark
+    def run():
+        if grouped:
+            problem.init_bag()
+
+        for _ in range(20):
+            problem.reset()
+            problem.full_penalty
+
+        problem.create_result_data()

glotaran/analysis/nnls.py

@@ -7,9 +7,7 @@
 from scipy.optimize import nnls
 
 
-def residual_nnls(
-    matrix: np.ndarray, data: np.ndarray
-) -> typing.Tuple[typing.List[str], np.ndarray]:
+def residual_nnls(matrix: np.ndarray, data: np.ndarray) -> typing.Tuple[np.ndarray, np.ndarray]:
     """Calculate the conditionally linear parameters and residual with the nnls method.
 
     nnls stands for 'non-negative least-squares'.