✨ Add parameter IO support for more formats supported by pandas

.github/CODEOWNERS

@@ -33,7 +33,7 @@ LICENSE                             @glotaran/pyglotaran_creators
 # builtin module:
 /glotaran/builtin/io/*              @glotaran/admins
 /glotaran/builtin/io/ascii          @jsnel @glotaran/maintainers
-/glotaran/builtin/io/csv            @glotaran/maintainers
+/glotaran/builtin/io/pandas         @glotaran/maintainers
 /glotaran/builtin/io/netCDF         @glotaran/maintainers
 /glotaran/builtin/io/sdt            @glotaran/maintainers
 /glotaran/builtin/megacomplexes/    @jsnel @joernweissenborn

.pre-commit-config.yaml

@@ -101,15 +101,15 @@ repos:
         args:
           - "--select=D,DAR"
         name: "flake8 lint docstrings"
-        files: "^glotaran/(plugin_system|utils|deprecation|testing|parameter|project|model/property.py)"
+        files: "^glotaran/(plugin_system|utils|deprecation|testing|parameter|project|model/property.py|builtin/io/pandas)"
         exclude: "docs|tests?/"
         additional_dependencies: [flake8-docstrings, darglint==1.8.0]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.931
     hooks:
       - id: mypy
-        files: "^glotaran/(plugin_system|utils|deprecation|testing|parameter|project|model/property.py)"
+        files: "^glotaran/(plugin_system|utils|deprecation|testing|parameter|project|model/property.py|builtin/io/pandas)"
         exclude: "docs"
         additional_dependencies: [types-all]
 

changelog.md

@@ -12,6 +12,7 @@
 
 - 👌🎨 Add proper repr for DatasetMapping (#957)
 - 👌 Add SavingOptions to save_result API (#966)
+- ✨ Add parameter IO support for more formats supported by pandas (#896)
 
 ### 🩹 Bug fixes
 

glotaran/builtin/io/pandas/__init__.py

@@ -0,0 +1 @@
+"""Pandas io package."""

glotaran/builtin/io/pandas/csv.py

@@ -0,0 +1,66 @@
+"""Module containing CSV io support."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from glotaran.io import ProjectIoInterface
+from glotaran.io import register_project_io
+from glotaran.parameter import ParameterGroup
+from glotaran.utils.io import safe_dataframe_fillna
+from glotaran.utils.io import safe_dataframe_replace
+
+
+@register_project_io(["csv"])
+class CsvProjectIo(ProjectIoInterface):
+    """Plugin for CSV data io."""
+
+    def load_parameters(self, file_name: str, sep: str = ",") -> ParameterGroup:
+        """Load parameters from CSV file.
+
+        Parameters
+        ----------
+        file_name : str
+            Name of file to be loaded.
+        sep: str
+            Other separators can be used optionally., by default ','
+
+        Returns
+        -------
+            :class:`ParameterGroup
+        """
+        df = pd.read_csv(file_name, skipinitialspace=True, na_values=["None", "none"], sep=sep)
+        safe_dataframe_fillna(df, "minimum", -np.inf)
+        safe_dataframe_fillna(df, "maximum", np.inf)
+        return ParameterGroup.from_dataframe(df, source=file_name)
+
+    def save_parameters(
+        self,
+        parameters: ParameterGroup,
+        file_name: str,
+        *,
+        sep: str = ",",
+        as_optimized: bool = True,
+        replace_infinfinity: bool = True,
+    ) -> None:
+        """Save a :class:`ParameterGroup` to a CSV file.
+
+        Parameters
+        ----------
+        parameters : ParameterGroup
+            Parameters to be saved to file.
+        file_name : str
+            File to write the parameters to.
+        sep: str
+            Other separators can be used optionally., by default ','
+        as_optimized : bool
+            Weather to include properties which are the result of optimization.
+        replace_infinfinity : bool
+            Weather to replace infinity values with empty strings.
+        """
+        df = parameters.to_dataframe(as_optimized=as_optimized)
+        if replace_infinfinity is True:
+            safe_dataframe_replace(df, "minimum", -np.inf, "")
+            safe_dataframe_replace(df, "maximum", np.inf, "")
+        df.to_csv(file_name, na_rep="None", index=False, sep=sep)

glotaran/builtin/io/pandas/test/data/reference_parameters.csv

@@ -0,0 +1,8 @@
+label,value,expression,minimum,maximum,non-negative,vary,standard-error
+pure_list.1,1.0,None,,,False,True,None
+pure_list.2,2.0,None,,,False,True,None
+list_with_options.1,3.0,None,,,False,False,None
+list_with_options.2,4.0,None,,,False,False,None
+verbose_list.all_defaults,5.0,None,,,False,True,None
+verbose_list.no_defaults,6.0,None,,,True,False,None
+verbose_list.expression_only,11.0,$verbose_list.all_defaults + $verbose_list.no_defaults,,,False,False,None

glotaran/builtin/io/pandas/test/data/reference_parameters.tsv

@@ -0,0 +1,8 @@
+label	value	expression	minimum	maximum	non-negative	vary	standard-error
+pure_list.1	1.0	None			False	True	None
+pure_list.2	2.0	None			False	True	None
+list_with_options.1	3.0	None			False	False	None
+list_with_options.2	4.0	None			False	False	None
+verbose_list.all_defaults	5.0	None			False	True	None
+verbose_list.no_defaults	6.0	None			True	False	None
+verbose_list.expression_only	11.0	$verbose_list.all_defaults + $verbose_list.no_defaults			False	False	None

glotaran/builtin/io/pandas/test/data/reference_parameters.yaml

@@ -0,0 +1,8 @@
+pure_list: [1.0, 2.0]
+
+list_with_options: [3.0, 4.0, {vary: False}]
+
+verbose_list:
+  - ["all_defaults", 5.0]
+  - ["no_defaults", 6.0, {non-negative: True, vary: False, minimum: -1, maximum: 1}]
+  - ["expression_only", {expr: $verbose_list.all_defaults + $verbose_list.no_defaults}]

glotaran/builtin/io/pandas/test/test_pandas_parameters.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+from pandas.testing import assert_frame_equal
+
+from glotaran.io import load_parameters
+from glotaran.io import save_parameters
+from glotaran.parameter import ParameterGroup
+
+PANDAS_TEST_DATA = Path(__file__).parent / "data"
+PATH_XLSX = PANDAS_TEST_DATA / "reference_parameters.xlsx"
+PATH_ODS = PANDAS_TEST_DATA / "reference_parameters.ods"
+PATH_CSV = PANDAS_TEST_DATA / "reference_parameters.csv"
+PATH_TSV = PANDAS_TEST_DATA / "reference_parameters.tsv"
+
+
+@pytest.fixture(scope="module")
+def yaml_reference() -> ParameterGroup:
+    """Fixture for yaml reference data."""
+    return load_parameters(PANDAS_TEST_DATA / "reference_parameters.yaml")
+
+
+@pytest.mark.parametrize("reference_path", (PATH_XLSX, PATH_ODS, PATH_CSV, PATH_TSV))
+def test_references(yaml_reference: ParameterGroup, reference_path: Path):
+    """References are the same"""
+    result = load_parameters(reference_path)
+    assert result == yaml_reference
+
+
+@pytest.mark.parametrize(
+    "format_name,reference_path",
+    (("xlsx", PATH_XLSX), ("ods", PATH_ODS), ("csv", PATH_CSV), ("tsv", PATH_TSV)),
+)
+def test_roundtrips(
+    yaml_reference: ParameterGroup, tmp_path: Path, format_name: str, reference_path: Path
+):
+    """Roundtrip via save and load have the same data."""
+    format_reference = load_parameters(reference_path)
+    parameter_path = tmp_path / f"test_parameters.{format_name}"
+    save_parameters(file_name=parameter_path, format_name=format_name, parameters=yaml_reference)
+    parameters_roundtrip = load_parameters(parameter_path)
+
+    assert parameters_roundtrip == yaml_reference
+    assert parameters_roundtrip == format_reference
+
+    if format_name in {"csv", "tsv"}:
+        assert parameter_path.read_text() == reference_path.read_text()
+
+        first_data_line = parameter_path.read_text().splitlines()[1]
+        sep = "," if format_name == "csv" else "\t"
+
+        assert f"{sep}-inf" not in first_data_line
+        assert f"{sep}inf" not in first_data_line
+    else:
+        assert_frame_equal(
+            pd.read_excel(parameter_path, na_values=["None", "none"]),
+            pd.read_excel(reference_path, na_values=["None", "none"]),
+        )
+
+
+@pytest.mark.parametrize("format_name", ("xlsx", "ods", "csv", "tsv"))
+def test_as_optimized_false(yaml_reference: ParameterGroup, tmp_path: Path, format_name: str):
+    """Column 'standard-error' is missing if as_optimized==False"""
+    parameter_path = tmp_path / f"test_parameters.{format_name}"
+    save_parameters(
+        file_name=parameter_path,
+        format_name=format_name,
+        parameters=yaml_reference,
+        as_optimized=False,
+    )
+
+    if format_name in {"csv", "tsv"}:
+        assert "standard-error" not in parameter_path.read_text().splitlines()[0]
+    else:
+        assert (
+            "standard-error"
+            not in pd.read_excel(parameter_path, na_values=["None", "none"]).columns
+        )
+
+
+@pytest.mark.parametrize("format_name,sep", (("csv", ","), ("tsv", "\t")))
+def test_replace_infinfinity(
+    yaml_reference: ParameterGroup, tmp_path: Path, format_name: str, sep: str
+):
+    """Column 'standard-error' is missing if as_optimized==False"""
+    parameter_path = tmp_path / f"test_parameters.{format_name}"
+    save_parameters(
+        file_name=parameter_path,
+        format_name=format_name,
+        parameters=yaml_reference,
+        replace_infinfinity=False,
+    )
+    df = pd.read_csv(parameter_path, sep=sep)
+    assert all(df["minimum"] == -np.inf)
+    assert all(df["maximum"] == np.inf)
+
+    first_data_line = parameter_path.read_text().splitlines()[1]
+    assert f"{sep}-inf" in first_data_line
+    assert f"{sep}inf" in first_data_line
+
+    assert load_parameters(parameter_path) == yaml_reference

glotaran/builtin/io/pandas/tsv.py

@@ -0,0 +1,62 @@
+"""Module containing TSV io support."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from glotaran.io import ProjectIoInterface
+from glotaran.io import load_parameters
+from glotaran.io import register_project_io
+from glotaran.io import save_parameters
+
+if TYPE_CHECKING:
+    from glotaran.parameter import ParameterGroup
+
+
+@register_project_io(["tsv"])
+class TsvProjectIo(ProjectIoInterface):
+    """Plugin for TSV data io."""
+
+    def load_parameters(self, file_name: str) -> ParameterGroup:
+        """Load parameters from TSV file.
+
+        Parameters
+        ----------
+        file_name : str
+            Name of file to be loaded.
+
+        Returns
+        -------
+            :class:`ParameterGroup
+        """
+        return load_parameters(file_name, format_name="csv", sep="\t")
+
+    def save_parameters(
+        self,
+        parameters: ParameterGroup,
+        file_name: str,
+        *,
+        as_optimized: bool = True,
+        replace_infinfinity: bool = True,
+    ) -> None:
+        """Save a :class:`ParameterGroup` to a TSV file.
+
+        Parameters
+        ----------
+        parameters : ParameterGroup
+            Parameters to be saved to file.
+        file_name : str
+            File to write the parameters to.
+        as_optimized : bool
+            Whether to include properties which are the result of optimization.
+        replace_infinfinity : bool
+            Weather to replace infinity values with empty strings.
+        """
+        save_parameters(
+            parameters,
+            file_name,
+            format_name="csv",
+            sep="\t",
+            as_optimized=as_optimized,
+            replace_infinfinity=replace_infinfinity,
+        )

glotaran/builtin/io/pandas/xlsx.py

@@ -0,0 +1,53 @@
+"""Module containing Excel like io support."""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+
+from glotaran.io import ProjectIoInterface
+from glotaran.io import register_project_io
+from glotaran.parameter import ParameterGroup
+from glotaran.utils.io import safe_dataframe_fillna
+from glotaran.utils.io import safe_dataframe_replace
+
+
+@register_project_io(["xlsx", "ods"])
+class ExcelProjectIo(ProjectIoInterface):
+    """Plugin for Excel like data io."""
+
+    def load_parameters(self, file_name: str) -> ParameterGroup:
+        """Load parameters from XLSX file.
+
+        Parameters
+        ----------
+        file_name : str
+            Name of file to be loaded.
+
+        Returns
+        -------
+            :class:`ParameterGroup
+        """
+        df = pd.read_excel(file_name, na_values=["None", "none"])
+        safe_dataframe_fillna(df, "minimum", -np.inf)
+        safe_dataframe_fillna(df, "maximum", np.inf)
+        return ParameterGroup.from_dataframe(df, source=file_name)
+
+    def save_parameters(
+        self, parameters: ParameterGroup, file_name: str, *, as_optimized: bool = True
+    ):
+        """Save a :class:`ParameterGroup` to a Excel file.
+
+        Parameters
+        ----------
+        parameters : ParameterGroup
+            Parameters to be saved to file.
+        file_name : str
+            File to write the parameters to.
+        as_optimized : bool
+            Whether to include properties which are the result of optimization.
+        """
+        df = parameters.to_dataframe(as_optimized=as_optimized)
+        safe_dataframe_replace(df, "minimum", -np.inf, "")
+        safe_dataframe_replace(df, "maximum", np.inf, "")
+        df.to_excel(file_name, na_rep="None", index=False)

glotaran/parameter/parameter_group.py

@@ -323,7 +323,13 @@ def to_csv(self, filename: str, delimiter: str = ",") -> None:
         delimiter : str
             Character to separate columns., by default ","
         """
-        save_parameters(self, file_name=filename, allow_overwrite=True, sep=delimiter)
+        save_parameters(
+            self,
+            file_name=filename,
+            allow_overwrite=True,
+            sep=delimiter,
+            replace_infinfinity=False,
+        )
 
     def add_parameter(self, parameter: Parameter | list[Parameter]):
         """Add a :class:`Parameter` to the group.