Feature: Generators (#866)

* Added model generators

* Added spectral decay model generator
* Refactored examples
* Moved examples to testing and removed old model generator.
* Updated quickstart.
* Update howto write plugin documentation notebook
Adapt to new examples->testing API
* Use builtin yml for generators
* Use _ in generator names.
* Rename argument of model generator to generator_name.
* Added naming consistency.
* Refactored test simulations.
* Prevent using positional arguments in generators.
* Updated changelog.

* 🔧 Update codeowners file to reflect removed examples folder
* 🔧 Disable quickstart test in examples 🧪

* 🩹📚 Fix changed import in docs
glotaran.testing.sequential_spectral_decay -> glotaran.testing.simulated_data.sequential_spectral_decay
* 🩹 Added __init __.py so glotaran.testing.simulated_data is a proper package
* 👌 Made arguments for generators keyword only
* 🧪 Reactivated quick-start integration test
* ✨ Added module_load_overwrite arg to deprecation functions for modules
* 🚧🗑️ Properly deprecated  'glotaran.examples'

Minor touch ups by:
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Sourcery AI <>

Significant contributions by:
Co-authored-by: Sebastian Weigand <[email protected]>
Co-authored-by: Joris Snellenburg <[email protected]>

.github/CODEOWNERS

@@ -41,8 +41,8 @@ LICENSE                             @glotaran/pyglotaran_creators
 # cli
 /glotaran/cli/                      @jsnel @glotaran/admins
 
-# examples
-/glotaran/examples/                 @jsnel @glotaran/maintainers
+# examples # Removed in PR #866
+# /glotaran/examples/                 @jsnel @glotaran/maintainers
 
 # io
 /glotaran/io/                       @jsnel @glotaran/maintainers

changelog.md

@@ -5,6 +5,7 @@
 ### ✨ Features
 
 - ✨ Add simple decay megacomplexes (#860)
+- ✨ Feature: Generators (#866)
 
 ### 👌 Minor Improvements:
 

docs/source/notebooks/plugin_system/plugin_howto_write_a_io_plugin.ipynb

@@ -167,9 +167,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from glotaran.examples.sequential import dataset\n",
     "from glotaran.io import load_dataset\n",
-    "from glotaran.io import save_dataset"
+    "from glotaran.io import save_dataset\n",
+    "from glotaran.testing.simulated_data.sequential_spectral_decay import DATASET as dataset"
    ]
   },
   {

docs/source/notebooks/quickstart/quickstart.ipynb

@@ -54,7 +54,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from glotaran.examples.sequential import dataset\n",
+    "from glotaran.testing.simulated_data.sequential_spectral_decay import DATASET as dataset\n",
     "\n",
     "dataset"
    ]

glotaran/__init__.py

@@ -1,4 +1,5 @@
 """Glotaran package __init__.py"""
+from glotaran.deprecation.deprecation_utils import deprecate_submodule
 from glotaran.deprecation.modules.glotaran_root import read_model_from_yaml
 from glotaran.deprecation.modules.glotaran_root import read_model_from_yaml_file
 from glotaran.deprecation.modules.glotaran_root import read_parameters_from_csv_file
@@ -10,6 +11,13 @@
 
 __version__ = "0.6.0.dev0"
 
+examples = deprecate_submodule(
+    deprecated_module_name="glotaran.examples",
+    new_module_name="glotaran.testing.simulated_data",
+    to_be_removed_in_version="0.8.0",
+    module_load_overwrite="glotaran.deprecation.modules.examples",
+)
+
 
 def __getattr__(attribute_name: str):
     from glotaran.deprecation.deprecation_utils import deprecate_module_attribute

glotaran/builtin/io/folder/test/test_folder_plugin.py

@@ -1,23 +1,27 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import Literal
 
 import pytest
 
+from glotaran.analysis.optimize import optimize
 from glotaran.io import save_result
-from glotaran.project.test.test_result import dummy_result  # noqa: F401
+from glotaran.project.result import Result
+from glotaran.testing.simulated_data.sequential_spectral_decay import SCHEME
 
-if TYPE_CHECKING:
-    from typing import Literal
 
-    from glotaran.project.result import Result
+@pytest.fixture(scope="session")
+def dummy_result():
+    """Dummy result for testing."""
+    print(SCHEME.data["dataset_1"])
+    yield optimize(SCHEME, raise_exception=True)
 
 
 @pytest.mark.parametrize("format_name", ("folder", "legacy"))
 def test_save_result_folder(
     tmp_path: Path,
-    dummy_result: Result,  # noqa: F811
+    dummy_result: Result,
     format_name: Literal["folder", "legacy"],
 ):
     """Check all files exist."""
@@ -34,9 +38,7 @@ def test_save_result_folder(
         "initial_parameters.csv",
         "optimized_parameters.csv",
         "parameter_history.csv",
-        "dataset1.nc",
-        "dataset2.nc",
-        "dataset3.nc",
+        "dataset_1.nc",
     ]
     for wanted in wanted_files:
         assert (result_dir / wanted).exists()
@@ -46,7 +48,7 @@ def test_save_result_folder(
 @pytest.mark.parametrize("format_name", ("folder", "legacy"))
 def test_save_result_folder_error_path_is_file(
     tmp_path: Path,
-    dummy_result: Result,  # noqa: F811
+    dummy_result: Result,
     format_name: Literal["folder", "legacy"],
 ):
     """Raise error if result_path is a file without extension and overwrite is true."""

glotaran/builtin/io/yml/test/test_save_model.py

@@ -2,57 +2,44 @@
 
 from typing import TYPE_CHECKING
 
-from glotaran.examples.sequential import model
 from glotaran.io import load_model
 from glotaran.io import save_model
+from glotaran.testing.simulated_data.sequential_spectral_decay import MODEL
 
 if TYPE_CHECKING:
     from pathlib import Path
 
 
-want = """\
-default_megacomplex: decay
+want = """default_megacomplex: decay-sequential
 dataset_groups:
   default:
     residual_function: variable_projection
     link_clp: null
-k_matrix:
-  k1:
-    matrix:
-      (s2, s1): kinetic.1
-      (s3, s2): kinetic.2
-      (s3, s3): kinetic.3
-initial_concentration:
-  j1:
-    compartments:
-      - s1
-      - s2
-      - s3
-    parameters:
-      - j.1
-      - j.0
-      - j.0
-    exclude_from_normalize: []
 irf:
-  irf1:
+  gaussian_irf:
     type: gaussian
     center: irf.center
     width: irf.width
     normalize: true
     backsweep: false
 megacomplex:
-  m1:
-    type: decay
+  megacomplex_sequential_decay:
+    type: decay-sequential
+    compartments:
+      - species_1
+      - species_2
+      - species_3
+    rates:
+      - rates.species_1
+      - rates.species_2
+      - rates.species_3
     dimension: time
-    k_matrix:
-      - k1
 dataset:
-  dataset1:
+  dataset_1:
     group: default
     megacomplex:
-      - m1
-    initial_concentration: j1
-    irf: irf1
+      - megacomplex_sequential_decay
+    irf: gaussian_irf
 """
 
 
@@ -62,7 +49,7 @@ def test_save_model(
     """Check all files exist."""
 
     model_path = tmp_path / "testmodel.yml"
-    save_model(file_name=model_path, format_name="yml", model=model)
+    save_model(file_name=model_path, format_name="yml", model=MODEL)
 
     assert model_path.is_file()
     assert model_path.read_text() == want

glotaran/builtin/io/yml/test/test_save_result.py

@@ -1,21 +1,29 @@
 from __future__ import annotations
 
+from dataclasses import replace
 from pathlib import Path
 from textwrap import dedent
-from typing import TYPE_CHECKING
+
+import pytest
 
 from glotaran import __version__
+from glotaran.analysis.optimize import optimize
 from glotaran.io import save_result
-from glotaran.project.test.test_result import dummy_result  # noqa: F401
+from glotaran.project.result import Result
+from glotaran.testing.simulated_data.sequential_spectral_decay import SCHEME
 
-if TYPE_CHECKING:
 
-    from glotaran.project.result import Result
+@pytest.fixture(scope="session")
+def dummy_result():
+    """Dummy result for testing."""
+    scheme = replace(SCHEME, maximum_number_function_evaluations=1)
+    print(scheme.data["dataset_1"])
+    yield optimize(scheme, raise_exception=True)
 
 
 def test_save_result_yml(
     tmp_path: Path,
-    dummy_result: Result,  # noqa: F811
+    dummy_result: Result,
 ):
     """Check all files exist."""
     expected = dedent(
@@ -25,16 +33,17 @@ def test_save_result_yml(
         termination_reason: The maximum number of function evaluations is exceeded.
         glotaran_version: {__version__}
         free_parameter_labels:
-          - '1'
-          - '2'
+          - rates.species_1
+          - rates.species_2
+          - rates.species_3
+          - irf.center
+          - irf.width
         scheme: scheme.yml
         initial_parameters: initial_parameters.csv
         optimized_parameters: optimized_parameters.csv
         parameter_history: parameter_history.csv
         data:
-          dataset1: dataset1.nc
-          dataset2: dataset2.nc
-          dataset3: dataset3.nc
+          dataset_1: dataset_1.nc
         """
     )
 
@@ -46,8 +55,7 @@ def test_save_result_yml(
     assert (result_dir / "result.yml").exists()
     assert (result_dir / "initial_parameters.csv").exists()
     assert (result_dir / "optimized_parameters.csv").exists()
-    assert (result_dir / "dataset1.nc").exists()
-    assert (result_dir / "dataset2.nc").exists()
-    assert (result_dir / "dataset3.nc").exists()
+    assert (result_dir / "dataset_1.nc").exists()
+
     # We can't check equality due to numerical fluctuations
     assert expected in (result_dir / "result.yml").read_text()

glotaran/builtin/io/yml/test/test_save_scheme.py

@@ -4,15 +4,15 @@
 
 import xarray as xr
 
-from glotaran.examples.sequential import dataset
-from glotaran.examples.sequential import model
-from glotaran.examples.sequential import parameter
 from glotaran.io import load_scheme
 from glotaran.io import save_dataset
 from glotaran.io import save_model
 from glotaran.io import save_parameters
 from glotaran.io import save_scheme
 from glotaran.project import Scheme
+from glotaran.testing.simulated_data.sequential_spectral_decay import DATASET
+from glotaran.testing.simulated_data.sequential_spectral_decay import MODEL
+from glotaran.testing.simulated_data.sequential_spectral_decay import PARAMETER
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -35,13 +35,13 @@
 
 
 def test_save_scheme(tmp_path: Path):
-    save_model(model, tmp_path / "m.yml")
-    save_parameters(parameter, tmp_path / "p.csv")
-    save_dataset(dataset, tmp_path / "d.nc")
+    save_model(MODEL, tmp_path / "m.yml")
+    save_parameters(PARAMETER, tmp_path / "p.csv")
+    save_dataset(DATASET, tmp_path / "d.nc")
     scheme = Scheme(
-        model,
-        parameter,
-        {"dataset_1": dataset},
+        MODEL,
+        PARAMETER,
+        {"dataset_1": DATASET},
     )
     scheme_path = tmp_path / "testscheme.yml"
     save_scheme(file_name=scheme_path, format_name="yml", scheme=scheme)

glotaran/builtin/io/yml/yml.py

@@ -4,6 +4,7 @@
 from typing import TYPE_CHECKING
 
 from ruamel.yaml import YAML
+from ruamel.yaml.compat import StringIO
 
 from glotaran.deprecation.modules.builtin_io_yml import model_spec_deprecations
 from glotaran.deprecation.modules.builtin_io_yml import scheme_spec_deprecations
@@ -83,7 +84,7 @@ def save_model(self, model: Model, file_name: str):
                     if isinstance(prop, dict) and any(isinstance(k, tuple) for k in prop):
                         keys = [f"({k[0]}, {k[1]})" for k in prop]
                         item[prop_name] = {f"{k}": v for k, v in zip(keys, prop.values())}
-        _write_dict(file_name, model_dict)
+        write_dict(model_dict, file_name=file_name)
 
     def load_parameters(self, file_name: str) -> ParameterGroup:
         """Create a ParameterGroup instance from the specs defined in a file.
@@ -111,7 +112,7 @@ def load_scheme(self, file_name: str) -> Scheme:
 
     def save_scheme(self, scheme: Scheme, file_name: str):
         scheme_dict = asdict(scheme, folder=Path(file_name).parent)
-        _write_dict(file_name, scheme_dict)
+        write_dict(scheme_dict, file_name=file_name)
 
     def load_result(self, result_path: str) -> Result:
         """Create a :class:`Result` instance from the specs defined in a file.
@@ -141,7 +142,7 @@ def save_result(self, result: Result, result_path: str):
         """
         save_result(result, Path(result_path).parent.as_posix(), format_name="folder")
         result_dict = asdict(result, folder=Path(result_path).parent)
-        _write_dict(result_path, result_dict)
+        write_dict(result_dict, file_name=result_path)
 
     def _load_yml(self, file_name: str) -> dict[str, Any]:
         yaml = YAML()
@@ -153,12 +154,17 @@ def _load_yml(self, file_name: str) -> dict[str, Any]:
         return spec
 
 
-def _write_dict(file_name: str, data: Mapping[str, Any]):
+def write_dict(data: Mapping[str, Any], file_name: str | None = None) -> str | None:
     yaml = YAML()
     yaml.representer.add_representer(type(None), _yaml_none_representer)
     yaml.indent(mapping=2, sequence=2, offset=2)
-    with open(file_name, "w") as f:
-        yaml.dump(data, f)
+    if file_name is not None:
+        with open(file_name, "w") as f:
+            yaml.dump(data, f)
+    else:
+        stream = StringIO()
+        yaml.dump(data, stream)
+        return stream.getvalue()
 
 
 def _yaml_none_representer(representer: BaseRepresenter, data: Mapping[str, Any]) -> ScalarNode: