From 62a66754669d1430ffba09f1de2375dd47a1e30f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 16:51:07 +0100 Subject: [PATCH 01/18] Silence warnings in run_notebook() --- genno/testing/jupyter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/genno/testing/jupyter.py b/genno/testing/jupyter.py index e1668f85..e2f8bd13 100644 --- a/genno/testing/jupyter.py +++ b/genno/testing/jupyter.py @@ -65,12 +65,16 @@ def run_notebook(nb_path, tmp_path, env=None, **kwargs): kwargs.setdefault("kernel_name", kernel or f"python{sys.version_info[0]}") kwargs.setdefault("timeout", 10) + # Set up environment + env = env or os.environ.copy() + env.setdefault("PYDEVD_DISABLE_FILE_VALIDATION", "1") + # Create a client and use it to execute the notebook client = NotebookClient(nb, **kwargs, resources=dict(metadata=dict(path=tmp_path))) # Execute the notebook. # `env` is passed from nbclient to jupyter_client.launcher.launch_kernel() - client.execute(env=env or os.environ.copy()) + client.execute(env=env) # Retrieve error information from cells errors = [ From 32b0256debbbf8f05786fceb995ce6f28cc923e9 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 16:53:22 +0100 Subject: [PATCH 02/18] Improve get_operator() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Cache return values; discard cache when .modules changes. - Don't use a result if getattr(…) triggers a warning. --- genno/core/computer.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/genno/core/computer.py b/genno/core/computer.py index 592969e0..9f71274f 100644 --- a/genno/core/computer.py +++ b/genno/core/computer.py @@ -1,6 +1,6 @@ import logging from collections import deque -from functools import partial +from functools import lru_cache, partial from importlib import import_module from inspect import signature from itertools import compress @@ -21,7 +21,7 @@ Union, cast, ) -from warnings import warn +from warnings import catch_warnings, warn import dask import pint @@ -153,13 +153,29 @@ def get_operator(self, name) -> Optional[Callable]: None If there is no callable with the given `name` in any of :attr:`modules`. """ + if not isinstance(name, str): + # `name` is not a string; can't be the name of a function/class/object + return None + + # Cached call with `name` guaranteed to be hashable + return self._get_operator(name) + + @lru_cache() + def _get_operator(self, name: str) -> Optional[Callable]: for module in reversed(self.modules): try: - return getattr(module, name) + # Retrieve the operator from `module` + with catch_warnings(record=True) as cw: + result = getattr(module, name) except AttributeError: continue # `name` not in this module - except TypeError: - return None # `name` is not a string; can't be the name of a function + else: + if len(cw): + continue # Some DeprecationWarning raised; don't use this import + else: + return result + + # Nothing found return None #: Alias of :meth:`get_operator`. @@ -220,6 +236,10 @@ def require_compat(self, pkg: Union[str, ModuleType]): if mod not in self.modules: self.modules.append(mod) + # Clear the lookup cache + # TODO also clear on manual changes to self.modules + self._get_operator.cache_clear() + # Add computations to the Computer def add(self, data, *args, **kwargs) -> Union[KeyLike, Tuple[KeyLike, ...]]: From 62dde281a1e78acc041e772d682166e7c84c7eae Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 16:59:41 +0100 Subject: [PATCH 03/18] Improve write_report() - Make a single-dispatch function for easier overloading. - Pass through keyword arguments; set index=False by default. - Add "header_comment" keyword argument. --- genno/operator.py | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/genno/operator.py b/genno/operator.py index 6e462b88..ae7f9510 100644 --- a/genno/operator.py +++ b/genno/operator.py @@ -5,7 +5,7 @@ import numbers import operator import re -from functools import partial, reduce +from functools import partial, reduce, singledispatch from itertools import chain from os import PathLike from pathlib import Path @@ -998,19 +998,55 @@ def add_sum( return c.add(key, func, qty, weights=weights, dimensions=dimensions, **kwargs) -def write_report(quantity: Quantity, path: Union[str, PathLike]) -> None: +def _format_header_comment(value: str) -> str: + if not len(value): + return value + + from textwrap import indent + + return indent(value + "\n", "# ", lambda line: True) + + +@singledispatch +def write_report( + quantity: pd.DataFrame, path: Union[str, PathLike], kwargs: Optional[dict] = None +) -> None: """Write a quantity to a file. Parameters ---------- - path : str or ~.pathlib.Path + quantity : + path : str or pathlib.Path Path to the file to be written. + kwargs : + Keyword arguments. For the default implementation, these are passed to + :meth:~pandas.DataFrame.to_csv` or :meth:~pandas.DataFrame.to_excel` (according + to `path`), except for: + + - "header_comment": valid only for `path` ending in :file:`.csv`. Multi-line + text that is prepended to the file, with comment characters ("# ") before + each line. """ path = Path(path) if path.suffix == ".csv": - quantity.to_dataframe().to_csv(path) + kwargs = kwargs or dict() + kwargs.setdefault("index", False) + + with open(path, "w") as f: + f.write(_format_header_comment(kwargs.pop("header_comment", ""))) + quantity.to_csv(f, **kwargs) elif path.suffix == ".xlsx": - quantity.to_dataframe().to_excel(path, merge_cells=False) + kwargs = kwargs or dict() + kwargs.setdefault("index", False) + kwargs.setdefault("merge_cells", False) + + quantity.to_excel(path, **kwargs) else: path.write_text(quantity) # type: ignore + + +@write_report.register +def _(quantity: Quantity, path, kwargs=None) -> None: + # Convert the Quantity to a pandas.DataFrame, then write + write_report(quantity.to_dataframe(), path, kwargs) From 6139a5fe291c8f5c029764297d19869201432a9a Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 17:00:27 +0100 Subject: [PATCH 04/18] Update test_write_report() --- genno/tests/core/test_computer.py | 2 +- genno/tests/test_operator.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/genno/tests/core/test_computer.py b/genno/tests/core/test_computer.py index 096056a9..ed10c868 100644 --- a/genno/tests/core/test_computer.py +++ b/genno/tests/core/test_computer.py @@ -798,7 +798,7 @@ def test_file_formats(test_data_path, tmp_path): # Write to CSV p3 = tmp_path / "output.csv" - c.write(k, p3) + c.write(k, p3, index=True) # Output is identical to input file, except for order assert sorted(p1.read_text().split("\n")) == sorted(p3.read_text().split("\n")) diff --git a/genno/tests/test_operator.py b/genno/tests/test_operator.py index b6652234..c55e40c8 100644 --- a/genno/tests/test_operator.py +++ b/genno/tests/test_operator.py @@ -58,7 +58,7 @@ def test_dims(op, expected_dims): y = c.add("y:b-c", None) key = c.add("z", op, x, y) - print(f"{key = }") + # print(f"{key = }") assert expected_dims == key.dims From d018fe5ba3ba142094760b3558d18e2fcbddf5a2 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 17:00:45 +0100 Subject: [PATCH 05/18] Make concat() single-dispatch --- genno/operator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/genno/operator.py b/genno/operator.py index ae7f9510..2a9ae8f6 100644 --- a/genno/operator.py +++ b/genno/operator.py @@ -406,6 +406,7 @@ def combine( return result +@singledispatch def concat(*objs: Quantity, **kwargs) -> Quantity: """Concatenate Quantity `objs`. From ef5110170831a159084854a29f5d823159a51b64 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 17:02:05 +0100 Subject: [PATCH 06/18] Pass keyword args through Computer.write() --- .pre-commit-config.yaml | 2 +- genno/core/computer.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d7bd035a..9ebfe2d7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.6.1 + rev: v1.7.0 hooks: - id: mypy additional_dependencies: diff --git a/genno/core/computer.py b/genno/core/computer.py index 9f71274f..608a6beb 100644 --- a/genno/core/computer.py +++ b/genno/core/computer.py @@ -295,7 +295,7 @@ def add(self, data, *args, **kwargs) -> Union[KeyLike, Tuple[KeyLike, ...]]: if func: try: # Use an implementation of Computation.add_task() - return func.add_tasks(self, *args, **kwargs) # type: ignore[union-attr] + return func.add_tasks(self, *args, **kwargs) # type: ignore [attr-defined] except (AttributeError, NotImplementedError): # Computation obj that doesn't implement .add_tasks(), or plain callable _partialed_func, kw = partial_split(func, kwargs) @@ -816,11 +816,11 @@ def visualize(self, filename, key=None, optimize_graph=False, **kwargs): return visualize(dsk, filename=filename, **kwargs) - def write(self, key, path): + def write(self, key, path, **kwargs): """Compute `key` and write the result directly to `path`.""" # Call the method directly without adding it to the graph key = self.check_keys(key)[0] - self.get_operator("write_report")(self.get(key), path) + self.get_operator("write_report")(self.get(key), path, kwargs) @property def unit_registry(self): From 467dd25ecfb86f406ab6cf76ac54be2002f085da Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 17:06:20 +0100 Subject: [PATCH 07/18] Single-dispatch .compat.pyam.{concat,write_report} Simplify these functions by removing the need to chain the basic operator. --- genno/compat/pyam/operator.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/genno/compat/pyam/operator.py b/genno/compat/pyam/operator.py index 418a837c..e62b7731 100644 --- a/genno/compat/pyam/operator.py +++ b/genno/compat/pyam/operator.py @@ -1,6 +1,5 @@ import logging from functools import partial -from os import PathLike from pathlib import Path from typing import TYPE_CHECKING, Callable, Collection, Iterable, Optional, Union from warnings import warn @@ -20,7 +19,7 @@ log = logging.getLogger(__name__) -__all__ = ["as_pyam", "concat", "write_report"] +__all__ = ["as_pyam"] @Operator.define() @@ -183,35 +182,35 @@ def add_as_pyam( return tuple(keys) if multi_arg else keys[0] -def concat(*args, **kwargs): - """Concatenate *args*, which must all be :class:`pyam.IamDataFrame`. +@genno.operator.concat.register +def _(*args: pyam.IamDataFrame, **kwargs) -> pyam.IamDataFrame: + """Concatenate `args`, which must all be :class:`pyam.IamDataFrame`. Otherwise, equivalent to :func:`genno.operator.concat`. """ - if isinstance(args[0], pyam.IamDataFrame): - # pyam.concat() takes an iterable of args - return pyam.concat(args, **kwargs) - else: - # genno.operator.concat() takes a variable number of positional arguments - return genno.operator.concat(*args, **kwargs) + # Use pyam.concat() top-level function + return pyam.concat(args, **kwargs) -def write_report(obj, path: Union[str, PathLike]) -> None: +@genno.operator.write_report.register +def _(quantity: pyam.IamDataFrame, path, kwargs=None) -> None: """Write `obj` to the file at `path`. If `obj` is a :class:`pyam.IamDataFrame` and `path` ends with ".csv" or ".xlsx", use :mod:`pyam` methods to write the file to CSV or Excel format, respectively. Otherwise, equivalent to :func:`genno.operator.write_report`. """ - if not isinstance(obj, pyam.IamDataFrame): - return genno.operator.write_report(obj, path) - path = Path(path) + if kwargs is not None and len(kwargs): + raise NotImplementedError( + "Keyword arguments to write_report(pyam.IamDataFrame, …)" + ) + if path.suffix == ".csv": - obj.to_csv(path) + quantity.to_csv(path) elif path.suffix == ".xlsx": - obj.to_excel(path) + quantity.to_excel(path) else: raise ValueError( f"pyam.IamDataFrame can be written to .csv or .xlsx, not {path.suffix}" From 8091302839a35ad26e939ab193e4fa873da0355f Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 17:07:01 +0100 Subject: [PATCH 08/18] Deprecate import of .compat.pyam{concat,write_report} --- genno/compat/pyam/operator.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/genno/compat/pyam/operator.py b/genno/compat/pyam/operator.py index e62b7731..597b94a1 100644 --- a/genno/compat/pyam/operator.py +++ b/genno/compat/pyam/operator.py @@ -215,3 +215,17 @@ def _(quantity: pyam.IamDataFrame, path, kwargs=None) -> None: raise ValueError( f"pyam.IamDataFrame can be written to .csv or .xlsx, not {path.suffix}" ) + + +def __getattr__(name: str): + if name in ("concat", "write_report"): + warn( + f"Importing {name!r} from genno.compat.pyam.operator; import from " + "genno.operator instead.", + DeprecationWarning, + 2, + ) + + return getattr(genno.operator, name) + else: + raise AttributeError(name) From 5515d727cf0ad94e19773275eb815d7146bd135d Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 17:07:40 +0100 Subject: [PATCH 09/18] =?UTF-8?q?Catch=20warnings=20in=20tests=20of=20conc?= =?UTF-8?q?at(pyam.IamDataFrame,=20=E2=80=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- genno/tests/compat/test_pyam.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/genno/tests/compat/test_pyam.py b/genno/tests/compat/test_pyam.py index c86353dc..9a3b9b35 100644 --- a/genno/tests/compat/test_pyam.py +++ b/genno/tests/compat/test_pyam.py @@ -252,13 +252,18 @@ def test_concat(dantzig_computer): year=2021, value=42.0, unit="kg" ) - result = operator.concat( - pyam.IamDataFrame(input), pyam.IamDataFrame(input.assign(year=2022)) - ) + with pytest.warns(DeprecationWarning): + result = operator.concat( + pyam.IamDataFrame(input), pyam.IamDataFrame(input.assign(year=2022)) + ) assert isinstance(result, pyam.IamDataFrame) # Other types pass through to base concat() - key = c.add("test", operator.concat, "fom:nl-t-ya", "vom:nl-t-ya", "tom:nl-t-ya") + with pytest.warns(DeprecationWarning): + key = c.add( + "test", operator.concat, "fom:nl-t-ya", "vom:nl-t-ya", "tom:nl-t-ya" + ) + c.get(key) From b7baecfa6de83175f10b5227d8cf3cda10ba7238 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Mon, 27 Nov 2023 17:08:02 +0100 Subject: [PATCH 10/18] Add all exceptions to API docs --- doc/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api.rst b/doc/api.rst index b9122bb2..fabd7b6a 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -307,7 +307,7 @@ Common :mod:`genno` usage, e.g. in :mod:`message_ix`, creates large, sparse data The goal is that all :mod:`genno`-based code, including built-in and user functions, can treat quantity arguments as if they were :class:`~xarray.DataArray`. .. automodule:: genno - :members: MissingKeyError + :members: ComputationError, KeyExistsError, MissingKeyError Operators ========= From 3e9067665328f41c7d30f2b2c7a54861bf38a8bf Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 11:18:40 +0100 Subject: [PATCH 11/18] Configure Sphinx to be nitpicky --- doc/conf.py | 49 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index b876dd0e..0572c13f 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -26,22 +26,21 @@ "sphinx.ext.viewcode", ] -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - # List of patterns, relative to source directory, that match files and directories to # ignore when looking for source files. This pattern also affects html_static_path and # html_extra_path. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] +nitpicky = True rst_prolog = """ .. role:: py(code) :language: python - -.. |KeyLike| replace:: :obj:`~genno.core.key.KeyLike` """ +# Paths that contain templates, relative to the current directory. +templates_path = ["_templates"] + def setup(app): """Modify the sphinx.ext.autodoc config to handle Operators as functions.""" @@ -89,19 +88,41 @@ def can_document_member(cls, member, membername, isattr, parent) -> bool: # -- Options for sphinx.ext.intersphinx ------------------------------------------------ intersphinx_mapping = { - "dask": ("https://docs.dask.org/en/stable/", None), + "dask": ("https://docs.dask.org/en/stable", None), "ixmp": ("https://docs.messageix.org/projects/ixmp/en/latest", None), - "joblib": ("https://joblib.readthedocs.io/en/latest/", None), - "graphviz": ("https://graphviz.readthedocs.io/en/stable/", None), + "joblib": ("https://joblib.readthedocs.io/en/latest", None), + "graphviz": ("https://graphviz.readthedocs.io/en/stable", None), "message_ix": ("https://docs.messageix.org/en/latest", None), - "pandas": ("https://pandas.pydata.org/docs/", None), - "pint": ("https://pint.readthedocs.io/en/stable/", None), - "plotnine": ("https://plotnine.readthedocs.io/en/stable/", None), - "pyam": ("https://pyam-iamc.readthedocs.io/en/stable/", None), - "python": ("https://docs.python.org/3/", None), + "message-ix-models": ("https://docs.messageix.org/projects/models/en/latest", None), + "nbclient": ("https://nbclient.readthedocs.io/en/latest", None), + "nbformat": ("https://nbformat.readthedocs.io/en/latest", None), + "numpy": ("https://numpy.org/doc/stable", None), + "pandas": ("https://pandas.pydata.org/docs", None), + "pint": ("https://pint.readthedocs.io/en/stable", None), + "plotnine": ("https://plotnine.readthedocs.io/en/stable", None), + "pyam": ("https://pyam-iamc.readthedocs.io/en/stable", None), + "python": ("https://docs.python.org/3", None), "sdmx1": ("https://sdmx1.readthedocs.io/en/stable", None), "sparse": ("https://sparse.pydata.org/en/stable", None), - "xarray": ("https://docs.xarray.dev/en/stable/", None), + "xarray": ("https://docs.xarray.dev/en/stable", None), +} + +# -- Options for sphinx.ext.napoleon --------------------------------------------------- + +napoleon_preprocess_types = True +napoleon_type_aliases = { + # Standard library + "callable": "typing.Callable", + "collection": "collections.abc.Collection", + "hashable": "collections.abc.Hashable", + "iterable": "collections.abc.Iterable", + "mapping": "collections.abc.Mapping", + "sequence": "collections.abc.Sequence", + "Path": "pathlib.Path", + # This package + "KeyLike": "genno.core.key.KeyLike", + # Others + "Code": "sdmx.model.common.Code", } # -- Options for sphinx.ext.todo ------------------------------------------------------- From b531070075a18c46402e9e7dae31d7944becf2e5 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 11:22:26 +0100 Subject: [PATCH 12/18] Address Sphinx nitpicks Also address nitpicks where docstrings are reused downstream via intersphinx. --- doc/api.rst | 4 +++ doc/cache.rst | 8 ++--- doc/compat-pyam.rst | 4 +-- doc/index.rst | 12 +++---- doc/whatsnew.rst | 34 +++++++++---------- genno/caching.py | 24 +++++++------- genno/compat/graphviz.py | 6 ++-- genno/compat/plotnine/plot.py | 24 +++++++------- genno/compat/pyam/operator.py | 38 ++++++++++++++-------- genno/compat/sdmx.py | 9 +++--- genno/compat/xarray.py | 9 ++++++ genno/config.py | 8 +++-- genno/core/attrseries.py | 4 +-- genno/core/computer.py | 59 ++++++++++++++++----------------- genno/core/describe.py | 3 +- genno/core/graph.py | 6 ++-- genno/core/key.py | 32 +++++++++--------- genno/core/operator.py | 13 ++++---- genno/core/quantity.py | 2 +- genno/core/sparsedataarray.py | 15 +++++---- genno/operator.py | 61 +++++++++++++++++++---------------- genno/testing/__init__.py | 27 ++++++++-------- genno/testing/jupyter.py | 8 ++--- genno/util.py | 4 +-- 24 files changed, 223 insertions(+), 191 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index fabd7b6a..e2216653 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -287,6 +287,7 @@ Top-level classes and functions .. autoclass:: genno.Quantity :members: + :inherited-members: pipe, shape, size The :class:`.Quantity` constructor converts its arguments to an internal, :class:`xarray.DataArray`-like data format: @@ -424,3 +425,6 @@ Utilities for testing .. automodule:: genno.testing :members: :exclude-members: parametrize_quantity_class + +.. automodule:: genno.testing.jupyter + :members: diff --git a/doc/cache.rst b/doc/cache.rst index 8b8bbcdd..4d4d0de8 100644 --- a/doc/cache.rst +++ b/doc/cache.rst @@ -31,7 +31,7 @@ A cache key is computed from: 1. the name of `func`. 2. the arguments to `func`, and -3. the compiled bytecode of `func` (see :func:`hash_code`). +3. the compiled bytecode of `func` (see :func:`.hash_code`). If a file exists in ``cache_path`` with a matching key, it is loaded and returned instead of calling `func`. @@ -61,12 +61,12 @@ Consider a function that loads a very large file, or performs some slow processi # … further processing … return Quantity(result) -We want to cache the result of :func:`slow_data_load`, but have the cache refreshed when the file contents change. +We want to cache the result of :py:`slow_data_load`, but have the cache refreshed when the file contents change. We do this using the `_extra_cache_key` argument to the function. This argument is not used in the function, but *does* affect the value of the cache key. When calling the function, pass some value that indicates whether the contents of `path` have changed. -One possibility is the modification time, via :meth:`.Path.stat`: +One possibility is the modification time, via :meth:`pathlib.Path.stat`: .. code-block:: python @@ -74,7 +74,7 @@ One possibility is the modification time, via :meth:`.Path.stat`: return slow_data_load(path, path.stat().st_mtime) Another possibility is to hash the entire file. -:func:`hash_contents` is provided for this purpose: +:func:`.hash_contents` is provided for this purpose: .. code-block:: python diff --git a/doc/compat-pyam.rst b/doc/compat-pyam.rst index 85805567..7de5b1a5 100644 --- a/doc/compat-pyam.rst +++ b/doc/compat-pyam.rst @@ -45,10 +45,8 @@ Pyam (:mod:`.compat.pyam`) as_pyam add_as_pyam - concat - write_report - .. .. autofunction:: as_pyam + This module also registers implementations of :func:`.concat` and :func:`.write_report` that handle :class:`pyam.IamDataFrame` objects. .. autofunction:: add_as_pyam diff --git a/doc/index.rst b/doc/index.rst index 05f7f6fa..4ae40554 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -4,7 +4,7 @@ **genno** is a Python package for describing and executing complex calculations on labelled, multi-dimensional data. It aims to make these calculations efficient, transparent, and easily validated as part of scientific research. -genno is built on high-quality Python data packages including ``dask``, ``xarray``, and ``pint``; and provides (current or planned) compatibility with packages including ``pandas``, ``matplotlib``, ``plotnine``, ``ixmp``, ``sdmx1``, and ``pyam``. +genno is built on high-quality Python data packages including :py:`dask`, :mod:`xarray`, and ```pint``; and provides (current or planned) compatibility with packages including :mod:`pandas`, :mod:`matplotlib`, :mod:`plotnine <.compat.plotnine>`, :mod:`ixmp`, :mod:`sdmx1 <.compat.sdmx>`, and :mod:`pyam <.compat.pyam>`. .. toctree:: :maxdepth: 2 @@ -22,9 +22,9 @@ Compatibility :mod:`.genno` provides built-in support for interaction with: -- :doc:`Plotnine ` (:mod:`.plotnine`), via :mod:`.compat.plotnine`. -- :doc:`Pyam ` (:mod:`.pyam`), via :mod:`.compat.pyam`. -- :doc:`SDMX ` (:mod:`.sdmx`), via :mod:`.compat.sdmx`. +- :doc:`Plotnine ` (:mod:`plotnine`), via :mod:`.compat.plotnine`. +- :doc:`Pyam ` (:mod:`pyam`), via :mod:`.compat.pyam`. +- :doc:`SDMX ` (:mod:`sdmx`), via :mod:`.compat.sdmx`. .. toctree:: :maxdepth: 1 @@ -37,8 +37,8 @@ Compatibility Packages that extend :mod:`genno` include: -- :mod:`ixmp.reporting` -- :mod:`message_ix.reporting` +- :mod:`ixmp.report` +- :mod:`message_ix.report` .. toctree:: :maxdepth: 2 diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 0f109f76..3ebf0048 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -123,7 +123,7 @@ v1.17.0 (2023-05-15) ==================== - Bug fix: genno v1.16.1 (:pull:`85`) introduced :class:`ValueError` for some usages of :func:`.computations.sum <.operator.sum>` (:pull:`88`). -- Provide typed signatures for :meth:`Quantity.bfill`, :meth:`~Quantity.cumprod`, :meth:`~Quantity.ffill` for the benefit of downstream applications (:pull:`88`). +- Provide typed signatures for :meth:`.Quantity.bfill`, :meth:`~.Quantity.cumprod`, :meth:`~.Quantity.ffill` for the benefit of downstream applications (:pull:`88`). - Ensure and test that :attr:`.Quantity.name` and :attr:`~.Quantity.units` pass through all :mod:`~genno.computations `, in particular :func:`~.operator.aggregate`, :func:`~.operator.convert_units`, and :func:`~.operator.sum` (:pull:`88`). - Simplify arithmetic operations (:func:`~.operator.div`, :func:`~.operator.mul`, :func:`~.operator.pow`) so they are agnostic as to the :class:`.Quantity` class in use (:pull:`88`). - Ensure :attr:`.AttrSeries.index` is always :class:`pandas.MultiIndex` (:pull:`88`). @@ -144,13 +144,13 @@ v1.16.0 (2023-04-29) - genno supports and is tested on Python 3.11 (:pull:`83`). - Update dependencies (:pull:`83`): - - General: :mod:`importlib_resources` (the independent backport of :mod:`importlib.resources`) is added for Python 3.9 and earlier. + - General: :py:`importlib_resources` (the independent backport of :mod:`importlib.resources`) is added for Python 3.9 and earlier. - ``genno[sparse]``: new set of optional dependencies, including :mod:`sparse`. Install this set in order to use :class:`.SparseDataArray` for :class:`.Quantity`. - Note that sparse depends on :mod:`numba`, and thus :mod:`llvmlite`, and both of these package can lag new Python versions by several months. + Note that sparse depends on :py:`numba`, and thus :py:`llvmlite`, and both of these package can lag new Python versions by several months. For example, as of this release, they do not yet support Python 3.11, and thus :mod:`sparse` and :class:`.SparseDataArray` can only be used with Python 3.10 and earlier. - - ``genno[tests]``: :mod:`ixmp` is removed; :mod:`jupyter` and :mod:`nbclient` are added. + - ``genno[tests]``: :mod:`ixmp` is removed; :py:`jupyter` and :py:`nbclient` are added. Testing utilities in :mod:`genno.testing.jupyter` are duplicated from :mod:`ixmp.testing.jupyter`. - Adjust :meth:`.AttrSeries.interp` for compatibility with pandas 2.0.0 (released 2023-04-03) (:pull:`81`). @@ -268,7 +268,7 @@ v1.9.1 (2022-01-27) Note that installing ``genno[pyam]`` (including via ``genno[compat]``) currently forces the installation of an old version of :mod:`pint`; version 0.17 or earlier. Users wishing to use :mod:`genno.compat.pyam` should first install ``genno[pyam]``, then ``pip install --upgrade pint`` to restore a recent version of pint (0.18 or newer) that is usable with genno. -- :func:`computations.concat` works with :class:`.AttrSeries` with misaligned dimensions (:pull:`53`). +- :func:`.computations.concat <.operator.concat>` works with :class:`.AttrSeries` with misaligned dimensions (:pull:`53`). - Improve typing of :class:`.Quantity` and :class:`.Computer` to help with using `mypy `_ on code that uses :mod:`genno` (:pull:`53`). v1.9.0 (2021-11-23) @@ -307,7 +307,7 @@ v1.7.0 (2021-07-22) v1.6.0 (2021-07-07) =================== -- Add :meth:`Key.permute_dims` (:pull:`47`). +- Add :py:`Key.permute_dims()` (:pull:`47`). - Improve performance of :meth:`.Computer.check_keys` (:pull:`47`). v1.5.2 (2021-07-06) @@ -323,9 +323,9 @@ v1.5.1 (2021-07-01) v1.5.0 (2021-06-27) =================== -- Adjust :meth:`.test_assign_coords` for xarray 0.18.2 (:pull:`43`). +- Adjust :func:`.test_assign_coords` for xarray 0.18.2 (:pull:`43`). - Make :attr:`.Key.dims` order-insensitive so that ``Key("foo", "ab") == Key("foo", "ba")`` (:pull:`42`); make corresponding changes to :class:`.Computer` (:pull:`44`). -- Fix “:class:`AttributeError`: 'COO' object has no attribute 'item'” on :meth:`SparseDataArray.item` (:pull:`41`). +- Fix “:class:`AttributeError`: 'COO' object has no attribute 'item'” on :meth:`.SparseDataArray.item` (:pull:`41`). v1.4.0 (2021-04-26) =================== @@ -339,18 +339,18 @@ v1.3.0 (2021-03-22) - Bump minimum version of :mod:`sparse` from 0.10 to 0.12 and adjust to changes in this version (:pull:`39`) - - Remove :meth:`.SparseDataArray.equals`, obviated by improvements in :mod:`sparse`. + - Remove :py:`SparseDataArray.equals()`, obviated by improvements in :mod:`sparse`. - Improve :class:`.AttrSeries` (:pull:`39`) - Implement :meth:`~.AttrSeries.drop_vars` and :meth:`~.AttrSeries.expand_dims`. - :meth:`~.AttrSeries.assign_coords` can relabel an entire dimension. - - :meth:`~.AttrSeries.sel` can accept :class:`.DataArray` indexers and rename/combine dimensions. + - :meth:`~.AttrSeries.sel` can accept :class:`~xarray.DataArray` indexers and rename/combine dimensions. v1.2.1 (2021-03-08) =================== -- Bug fix: Provide abstract :class:`.Quantity.to_series` method for type checking in packages that depend on :mod:`genno`. +- Bug fix: Provide abstract :meth:`.Quantity.to_series` method for type checking in packages that depend on :mod:`genno`. v1.2.0 (2021-03-08) =================== @@ -377,15 +377,15 @@ v1.1.0 (2021-02-16) v1.0.0 (2021-02-13) =================== -- Adjust for usage by :mod:`ixmp.reporting` and :mod:`message_ix.reporting` (:pull:`28`): +- Adjust for usage by :mod:`ixmp.reporting ` and :mod:`message_ix.reporting ` (:pull:`28`): - Reduce minimum Python version to 3.6. This is lower than the minimum version for xarray (3.7), but matches ixmp, etc. - - Remove :mod:`compat.ixmp`; this code has been moved to :mod:`ixmp.reporting`, replacing what was there. - Likewise, remove :mod:`compat.message_ix`. + - Remove submodule :py:`compat.ixmp`; this code has been moved to :mod:`ixmp.reporting `, replacing what was there. + Likewise, remove submodule :py:`compat.message_ix`. - Simplify the form & parsing of ``iamc:`` section entries in configuration files: - - Remove unused feature to add :func:`group_sum` to the chain of tasks. + - Remove unused feature to add :py:`group_sum()` to the chain of tasks. - Keys now conform more closely to the arguments of :meth:`.Computer.convert_pyam`. - Move argument-checking from :func:`.as_pyam` to :meth:`.convert_pyam()`. @@ -416,11 +416,11 @@ v0.2.0 (2021-01-18) ------------------- - Increase test coverage to 100% (:pull:`12`). -- Port code from :mod:`message_ix.reporting` (:pull:`11`). +- Port code from :mod:`message_ix.reporting ` (:pull:`11`). - Add :mod:`.compat.pyam`. - Add a `name` parameter to :func:`.load_file`. v0.1.0 (2021-01-10) ------------------- -- Initial code port from :mod:`ixmp.reporting`. +- Initial code port from :mod:`ixmp.reporting `. diff --git a/genno/caching.py b/genno/caching.py index 1fa0a2e4..0481f1f0 100644 --- a/genno/caching.py +++ b/genno/caching.py @@ -53,7 +53,7 @@ def ignore(cls, *types): @classmethod def register(cls, func): - """Register a `func` to serialize a type not handled by :class:`.JSONEncoder`. + """Register `func` to serialize a type not handled by :class:`json.JSONEncoder`. `func` should return a type that *is* handled by JSONEncoder; see the docs. @@ -69,12 +69,12 @@ def register(cls, func): return _encode.register(func) def default(self, o): - """For `o`, return an object serializable by the base :class:`.JSONEncoder`. + """For `o`, return an object serializable by the base :class:`json.JSONEncoder`. - :class:`pathlib.Path`: the string representation of `o`. - - :class:`code` objects (from Python's built-in :mod:`inspect` module), e.g. - a function or lambda: :class:`blake2b` hash of the object's bytecode and its - serialized constants. + - :ref:`python:code-objects` (from Python's built-in :mod:`inspect` module), for + instance a function or lambda: :func:`~hashlib.blake2b` hash of the object's + bytecode and its serialized constants. .. warning:: This is not 100% guaranteed to change if the operation of `o` (or other code called in turn by `o`) changes. If relying on this behaviour, @@ -102,7 +102,7 @@ def default(self, o): def hash_args(*args, **kwargs): - """Return a 20-character :class:`blake2b` hex digest of `args` and `kwargs`. + """Return a 20-character :func:`hashlib.blake2b` hex digest of `args` and `kwargs`. Used by :func:`.decorate`. @@ -121,7 +121,7 @@ def hash_args(*args, **kwargs): def hash_code(func: Callable) -> str: - """Return the :class:`.blake2b` hex digest of the compiled bytecode of `func`. + """Return the :func:`hashlib.blake2b` hex digest of the compiled bytecode of `func`. See also -------- @@ -133,11 +133,11 @@ def hash_code(func: Callable) -> str: def hash_contents(path: Union[Path, str], chunk_size=65536) -> str: - """Return the :class:`.blake2b` hex digest of the contents of the file at `path`. + """Return the :func:`hashlib.blake2b` hex digest the file contents of `path`. Parameters ---------- - chunk_size : int, *optional* + chunk_size : int, optional Read the file in chunks of this size; default 64 kB. """ with Path(path).open("rb") as f: @@ -154,13 +154,13 @@ def decorate( Parameters ---------- - computer : Computer, *optional* + computer : .Computer, optional If supplied, the ``config`` dictionary stored in the Computer is used to look up values for `cache_path` and `cache_skip`, at the moment when `func` is called. - cache_path : os.Pathlike, *optional* + cache_path : os.PathLike, optional Directory in which to store cache files. - cache_skip : bool, *optional* + cache_skip : bool, optional If :obj:`True`, ignore existing cache entries and overwrite them with new values from `func`. diff --git a/genno/compat/graphviz.py b/genno/compat/graphviz.py index 4e161eee..6906eaf0 100644 --- a/genno/compat/graphviz.py +++ b/genno/compat/graphviz.py @@ -51,11 +51,11 @@ def visualize( ---------- dsk : The graph to display. - filename : Path or str, *optional* + filename : Path or str, optional The name of the file to write to disk. If the file name does not have a suffix, ".png" is used by default. If `filename` is :data:`None`, no file is written, and dask communicates with :program:`dot` using only pipes. - format : {'png', 'pdf', 'dot', 'svg', 'jpeg', 'jpg'}, *optional* + format : {'png', 'pdf', 'dot', 'svg', 'jpeg', 'jpg'}, optional Format in which to write output file, if not given by the suffix of `filename`. Default "png". data_attributes : @@ -73,7 +73,7 @@ def visualize( edge_attr : Mapping of (attribute, value) pairs set for all edges. Passed directly to :class:`.graphviz.Digraph`. - collapse_outputs : bool, *optional* + collapse_outputs : bool, optional Omit nodes for keys that are the output of intermediate calculations. kwargs : All other keyword arguments are added to `graph_attr`. diff --git a/genno/compat/plotnine/plot.py b/genno/compat/plotnine/plot.py index dba5792c..b7377197 100644 --- a/genno/compat/plotnine/plot.py +++ b/genno/compat/plotnine/plot.py @@ -1,21 +1,19 @@ import logging from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Hashable, Sequence +from typing import Hashable, Sequence from warnings import warn import plotnine as p9 +from genno.core.computer import Computer +from genno.core.key import KeyLike from genno.core.quantity import Quantity -if TYPE_CHECKING: - from genno.core.computer import Computer - from genno.core.key import KeyLike - log = logging.getLogger(__name__) class Plot(ABC): - """Class for plotting using :mod:`plotnine`.""" + """Class for plotting using :doc:`plotnine `.""" #: Filename base for saving the plot. basename = "" @@ -84,7 +82,7 @@ def make_task(cls, *inputs): Parameters ---------- - inputs : sequence of :class:`.Key`, :class:`str`, or other hashable, *optional* + *inputs : `.Key` or str or hashable, optional If provided, overrides the :attr:`inputs` property of the class. Returns @@ -106,8 +104,8 @@ def make_task(cls, *inputs): @classmethod def add_tasks( - cls, c: "Computer", key: "KeyLike", *inputs, strict: bool = False - ) -> "KeyLike": + cls, c: Computer, key: KeyLike, *inputs, strict: bool = False + ) -> KeyLike: """Add a task to `c` to generate and save the Plot. Analogous to :meth:`.Operator.add_tasks`. @@ -125,8 +123,8 @@ def generate(self, *args, **kwargs): Parameters ---------- - args : sequence of :class:`pandas.DataFrame` - Because :mod:`plotnine` operates on pandas data structures, :meth:`save` - automatically converts :obj:`Quantity` before being provided to - :meth:`generate`. + args : sequence of pandas.DataFrame + Because :doc:`plotnine ` operates on pandas data structures, + :meth:`save` automatically converts :obj:`.Quantity` before they are passed + to :meth:`generate`. """ diff --git a/genno/compat/pyam/operator.py b/genno/compat/pyam/operator.py index 597b94a1..04ecf93a 100644 --- a/genno/compat/pyam/operator.py +++ b/genno/compat/pyam/operator.py @@ -1,7 +1,15 @@ import logging from functools import partial from pathlib import Path -from typing import TYPE_CHECKING, Callable, Collection, Iterable, Optional, Union +from typing import ( + TYPE_CHECKING, + Callable, + Collection, + Iterable, + Mapping, + Optional, + Union, +) from warnings import warn import pyam @@ -27,7 +35,7 @@ def as_pyam( scenario, quantity: "Quantity", *, - rename=dict(), + rename=Mapping[str, str], collapse: Optional[Callable] = None, replace=dict(), drop: Union[Collection[str], str] = "auto", @@ -54,22 +62,24 @@ def as_pyam( Parameters ---------- scenario : - Any object with :attr:`model` and :attr:`scenario` attributes of type - :class:`str`, for instance an :class:`ixmp.Scenario`. - rename : dict (str -> str), *optional* - Mapping from dimension names in `quantity` to column names; either IAMC - dimension names, or others that are consumed by `collapse`. - collapse : callable, *optional* + Any object with :py:`model` and :py:`scenario` attributes of type :class:`str`, + for instance an :class:`ixmp.Scenario` or + :class:`~message_ix_models.util.scenarioinfo.ScenarioInfo`. + rename : dict, optional + Mapping from dimension names in `quantity` (:class:`str`) to column names + (:class:`str`); either IAMC dimension names, or others that are consumed by + `collapse`. + collapse : callable, optional Function that takes a :class:`pandas.DataFrame` and returns the same type. This function **may** collapse 2 or more dimensions, for example to construct labels for the IAMC ``variable`` dimension, or any other. - replace : *optional* + replace : optional Values to be replaced and their replaced. Passed directly to :meth:`pandas.DataFrame.replace`. - drop : str or collection of str, *optional* + drop : str or collection of str, optional Columns to drop. Passed to :func:`.util.drop`, so if not given, all non-IAMC columns are dropped. - unit : str, *optional* + unit : str, optional Label for the IAMC ``unit`` dimension. Passed to :func:`~.pyam.util.clean_units`. @@ -130,9 +140,9 @@ def add_as_pyam( Parameters ---------- - quantities : str or Key or list of (str, Key) + quantities : str or .Key or list of str or .Key Keys for quantities to transform. - tag : str, *optional* + tag : str, optional Tag to append to new Keys. Other parameters @@ -142,7 +152,7 @@ def add_as_pyam( Returns ------- - list of Key + list of .Key Each task converts a :class:`.Quantity` into a :class:`pyam.IamDataFrame`. """ # Handle single vs. iterable of inputs diff --git a/genno/compat/sdmx.py b/genno/compat/sdmx.py index d904cd59..21f6696d 100644 --- a/genno/compat/sdmx.py +++ b/genno/compat/sdmx.py @@ -1,11 +1,12 @@ from typing import TYPE_CHECKING, Iterable, List, Mapping, Optional, Union if TYPE_CHECKING: - from sdmx.model.common import Code, Codelist + import sdmx.model.common def codelist_to_groups( - codes: Union["Codelist", Iterable["Code"]], dim: Optional[str] = None + codes: Union["sdmx.model.common.Codelist", Iterable["sdmx.model.common.Code"]], + dim: Optional[str] = None, ) -> Mapping[str, Mapping[str, List[str]]]: """Convert `codes` into a mapping from parent items to their children. @@ -16,14 +17,14 @@ def codelist_to_groups( codes Either a :class:`sdmx.Codelist ` object or any iterable of :class:`sdmx.Code `. - dim : str, *optional* + dim : str, optional Dimension to aggregate. If `codes` is a code list and `dim` is not given, the ID of the code list is used; otherwise `dim` must be supplied. """ from sdmx.model.common import Codelist if isinstance(codes, Codelist): - items: Iterable["Code"] = codes.items.values() + items: Iterable["sdmx.model.common.Code"] = codes.items.values() dim = dim or codes.id else: items = codes diff --git a/genno/compat/xarray.py b/genno/compat/xarray.py index 7cf8331e..564c1a52 100644 --- a/genno/compat/xarray.py +++ b/genno/compat/xarray.py @@ -85,6 +85,7 @@ def attrs(self) -> Dict[Any, Any]: @property def data(self) -> Any: + """Like :attr:`xarray.DataArray.data`.""" return NotImplemented @property @@ -99,10 +100,12 @@ def dims(self) -> Tuple[Hashable, ...]: @property def shape(self) -> Tuple[int, ...]: + """Like :attr:`xarray.DataArray.shape`.""" return NotImplemented @property def size(self) -> int: + """Like :attr:`xarray.DataArray.size`.""" return NotImplemented def assign_coords( @@ -122,6 +125,7 @@ def astype( copy=None, keep_attrs=True, ): + """Like :meth:`xarray.DataArray.astype`.""" ... def bfill( @@ -129,6 +133,7 @@ def bfill( dim: Hashable, limit: Optional[int] = None, ): + """Like :meth:`xarray.DataArray.bfill`.""" ... def copy( @@ -146,6 +151,7 @@ def cumprod( keep_attrs: Optional[bool] = None, **kwargs: Any, ): + """Like :meth:`xarray.DataArray.cumprod`.""" ... def drop_vars( @@ -169,6 +175,7 @@ def ffill( dim: Hashable, limit: Optional[int] = None, ): + """Like :meth:`xarray.DataArray.ffill`.""" return NotImplemented def groupby( @@ -198,6 +205,7 @@ def pipe( *args: Any, **kwargs: Any, ) -> T: + """Like :meth:`xarray.DataArray.pipe`.""" return NotImplemented def rename( @@ -226,6 +234,7 @@ def shift( fill_value: Any = None, **shifts_kwargs: int, ): + """Like :attr:`xarray.DataArray.shift`.""" ... def sum( diff --git a/genno/config.py b/genno/config.py index 3e323163..7e158d9c 100644 --- a/genno/config.py +++ b/genno/config.py @@ -43,7 +43,7 @@ def configure(path: Optional[Union[Path, str]] = None, **config): Parameters ---------- - path : pathlib.Path, *optional* + path : pathlib.Path, optional Path to a configuration file in JSON or YAML format. **config : Configuration keys/sections and values. @@ -62,11 +62,11 @@ def handles(section_name: str, iterate: bool = True, discard: bool = True): section_name: str The name of the configuration section to handle. Using a name already present in :data:`HANDLERS` overrides that handler. - iterate : bool, *optional* + iterate : bool, optional If :obj:`True`, the handler is called once for each item (either list item, or (key, value) tuple) in the section. If :obj:`False`, the entire section contents, of whatever type, are passed to tha handler. - discard : bool, *optional* + discard : bool, optional If :obj:`True`, configuration section data is discarded after the handler is called. If :obj:`False`, the data is retained and stored on the Computer. """ @@ -264,6 +264,8 @@ def files(c: Computer, info): def general(c: Computer, info): """Handle one entry from the ``general:`` config section.""" # Inputs + # TODO allow to specify a more narrow key and *not* have infer_keys applied; perhaps + # using "*" inputs = c.infer_keys(info.get("inputs", [])) if info["comp"] in ("mul", "product"): diff --git a/genno/core/attrseries.py b/genno/core/attrseries.py index 29d13705..27c86c5b 100644 --- a/genno/core/attrseries.py +++ b/genno/core/attrseries.py @@ -99,10 +99,10 @@ class AttrSeries(pd.Series, Quantity): Parameters ---------- - units : str or pint.Unit, *optional* + units : str or pint.Unit, optional Set the units attribute. The value is converted to :class:`pint.Unit` and added to `attrs`. - attrs : :class:`~collections.abc.Mapping`, *optional* + attrs : :class:`~collections.abc.Mapping`, optional Set the :attr:`~pandas.Series.attrs` of the AttrSeries. This attribute was added in `pandas 1.0 `_, but is not currently supported by the Series constructor. diff --git a/genno/core/computer.py b/genno/core/computer.py index 608a6beb..dc3b3727 100644 --- a/genno/core/computer.py +++ b/genno/core/computer.py @@ -108,9 +108,9 @@ def configure( Parameters ---------- - path : .Path, *optional* + path : pathlib.Path, optional Path to a configuration file in JSON or YAML format. - fail : "raise" or str or :mod:`logging` level, *optional* + fail : "raise" or str or :mod:`logging` level, optional Passed to :meth:`.add_queue`. If not "raise", then log messages are generated for config handlers that fail. The Computer may be only partially configured. @@ -149,7 +149,7 @@ def get_operator(self, name) -> Optional[Callable]: Returns ------- - .callable + callable None If there is no callable with the given `name` in any of :attr:`modules`. """ @@ -251,7 +251,7 @@ def add(self, data, *args, **kwargs) -> Union[KeyLike, Tuple[KeyLike, ...]]: Returns ------- - |KeyLike| or tuple of |KeyLike| + KeyLike or tuple of KeyLike Some or all of the keys added to the Computer. See also @@ -342,12 +342,13 @@ def add_queue( Parameters ---------- - queue : iterable of 2- or N-:class:`tuple` - The members of each tuple are the arguments (:class:`tuple`) and, - optionally, keyword arguments (e.g :class:`dict`) to :meth:`add`. - max_tries : int, *optional* + queue : iterable of tuple + Each item is either a N-:class:`tuple` of positional arguments to + :meth:`add`, or a 2-:class:`tuple` of (:class:`.tuple` of positional + arguments, :class:`dict` of keyword arguments). + max_tries : int, optional Retry adding elements up to this many times. - fail : "raise" or str or :mod:`logging` level, *optional* + fail : "raise" or str or :mod:`logging` level, optional Action to take when a computation from `queue` cannot be added after `max_tries`: "raise" an exception, or log messages on the indicated level and continue. @@ -434,10 +435,10 @@ def add_single( A string, Key, or other value identifying the output of `computation`. computation : object Any computation. See :attr:`graph`. - strict : bool, *optional* + strict : bool, optional If True, `key` must not already exist in the Computer, and any keys referred to by `computation` must exist. - index : bool, *optional* + index : bool, optional If True, `key` is added to the index as a full-resolution key, so it can be later retrieved with :meth:`full_key`. @@ -503,7 +504,7 @@ def apply( Parameters ---------- - generator : .callable + generator : callable Function to apply to `keys`. This function **may** take a first positional argument annotated with :class:`.Computer` or a subtype; if so, then it is provided with a reference to `self`. @@ -608,7 +609,7 @@ def get(self, key=None): Parameters ---------- - key : str, *optional* + key : str, optional If not provided, :attr:`default_key` is used. Raises @@ -646,7 +647,7 @@ def get(self, key=None): # Convenience methods for the graph and its keys def keys(self): - """Return the keys of :attr:`graph`.""" + """Return the keys of :attr:`~genno.Computer.graph`.""" return self.graph.keys() def full_key(self, name_or_key: KeyLike) -> KeyLike: @@ -676,16 +677,16 @@ def check_keys( Parameters ---------- - keys : |KeyLike| + keys : KeyLike Some :class:`Keys ` or strings. - predicate : callable, *optional* + predicate : callable, optional Function to run on each of `keys`; see below. - action : "raise" or any other value + action : "raise" or str Action to take on missing `keys`. Returns ------- - list of |KeyLike| + list of KeyLike One item for each item ``k`` in `keys`: 1. ``k`` itself, unchanged, if `predicate` is given and ``predicate(k)`` @@ -748,16 +749,16 @@ def infer_keys( Parameters ---------- - key_or_keys : |KeyLike| or list of |KeyLike| - dims : list of str, *optional* + key_or_keys : KeyLike or list of KeyLike + dims : list of str, optional Drop all but these dimensions from the returned key(s). Returns ------- - |KeyLike| - If `key_or_keys` is a single |KeyLike|. - list of |KeyLike| - If `key_or_keys` is an iterable of |KeyLike|. + KeyLike + If `key_or_keys` is a single KeyLike. + list of KeyLike + If `key_or_keys` is an iterable of KeyLike. """ single = isinstance(key_or_keys, (Key, Hashable)) keys = [key_or_keys] if single else tuple(cast(Iterable, key_or_keys)) @@ -852,7 +853,7 @@ def add_product(self, *args, **kwargs): """Deprecated. .. deprecated:: 1.18.0 - Instead use :func:`.add_mul` via: + Instead use :func:`.add_binop` via: .. code-block:: python @@ -902,13 +903,13 @@ def aggregate( quantity. dims_or_groups: str or iterable of str or dict Name(s) of the dimension(s) to sum over, or nested dict. - weights : :class:`xarray.DataArray`, *optional* + weights : :class:`xarray.DataArray`, optional Weights for weighted aggregation. - keep : bool, *optional* + keep : bool, optional Passed to :meth:`operator.aggregate `. - sums : bool, *optional* + sums : bool, optional Passed to :meth:`add`. - fail : str or int, *optional* + fail : str or int, optional Passed to :meth:`add_queue` via :meth:`add`. Returns diff --git a/genno/core/describe.py b/genno/core/describe.py index d1a27182..8b7e234f 100644 --- a/genno/core/describe.py +++ b/genno/core/describe.py @@ -83,7 +83,8 @@ def label(arg, max_length=MAX_ITEM_LENGTH) -> str: The label depends on the type of `arg`: - :class:`.xarray.DataArray`: the first line of the string representation. - - :func:`.partial` object: a less-verbose version that omits None arguments. + - :func:`functools.partial` object: a less-verbose version that omits None + arguments. - Item protected with :func:`.dask.core.quote`: its literal value. - A callable, e.g. a function: its name. - Anything else: its :class:`str` representation. diff --git a/genno/core/graph.py b/genno/core/graph.py index f867b77d..d453e7e8 100644 --- a/genno/core/graph.py +++ b/genno/core/graph.py @@ -80,12 +80,14 @@ def __contains__(self, item) -> bool: return False def pop(self, *args): + """Overload :meth:`dict.pop` to also call :meth:`_deindex`.""" try: return super().pop(*args) finally: self._deindex(args[0]) def update(self, arg=None, **kwargs): + """Overload :meth:`dict.pop` to also call :meth:`_index`.""" if isinstance(arg, (Sequence, Generator)): arg0, arg1 = tee(arg) arg_keys = map(itemgetter(0), arg0) @@ -114,14 +116,14 @@ def infer( Parameters ---------- - dims : list of str, *optional* + dims : list of str, optional Drop all but these dimensions from the returned key(s). Returns ------- str If `key` is not found in the Graph. - Key + .Key `key` with either its full dimensions (cf. :meth:`full_key`) or, if `dims` are given, with only these dims. """ diff --git a/genno/core/key.py b/genno/core/key.py index ff8c1be5..e384cfbb 100644 --- a/genno/core/key.py +++ b/genno/core/key.py @@ -113,22 +113,6 @@ def from_str_or_key( ) -> "Key": """Return a new Key from *value*. - Parameters - ---------- - value : str or Key - Value to use to generate a new Key. - drop : list of str or :obj:`True`, *optional* - Existing dimensions of *value* to drop. See :meth:`drop`. - append : list of str, *optional*. - New dimensions to append to the returned Key. See :meth:`append`. - tag : str, *optional* - Tag for returned Key. If *value* has a tag, the two are joined - using a '+' character. See :meth:`add_tag`. - - Returns - ------- - :class:`Key` - .. versionchanged:: 1.18.0 Calling :meth:`from_str_or_key` with a single argument is no longer @@ -141,6 +125,22 @@ def from_str_or_key( k1 = Key("foo:a-b-c:t1", drop="b", append="d", tag="t2") k2 = Key("foo:a-b-c:t1").drop("b").append("d)" + + Parameters + ---------- + value : str or .Key + Value to use to generate a new Key. + drop : list of str or :obj:`True`, optional + Existing dimensions of *value* to drop. See :meth:`drop`. + append : list of str, optional + New dimensions to append to the returned Key. See :meth:`append`. + tag : str, optional + Tag for returned Key. If *value* has a tag, the two are joined + using a '+' character. See :meth:`add_tag`. + + Returns + ------- + :class:`Key` """ base = cls(value) diff --git a/genno/core/operator.py b/genno/core/operator.py index 63d9f656..f999adf0 100644 --- a/genno/core/operator.py +++ b/genno/core/operator.py @@ -1,11 +1,10 @@ from functools import update_wrapper from inspect import signature -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Optional, Tuple, Union +from typing import Any, Callable, ClassVar, Dict, Optional, Tuple, Union from warnings import warn -if TYPE_CHECKING: - from .computer import Computer - from .key import KeyLike +from .computer import Computer +from .key import KeyLike class Operator: @@ -70,7 +69,7 @@ def define( Parameters ---------- - helper : Callable, *optional* + helper : Callable, optional Equivalent to calling :meth:`helper` on the Operator instance. """ @@ -111,13 +110,13 @@ def decorator(func: Callable) -> "Operator": return decorator def helper( - self, func: Callable[..., Union["KeyLike", Tuple["KeyLike", ...]]] + self, func: Callable[..., Union[KeyLike, Tuple[KeyLike, ...]]] ) -> Callable: """Register `func` as the convenience method for adding task(s).""" self.__class__._add_tasks = staticmethod(func) return func - def add_tasks(self, c: "Computer", *args, **kwargs) -> Tuple["KeyLike", ...]: + def add_tasks(self, c: "Computer", *args, **kwargs) -> Tuple[KeyLike, ...]: """Invoke :attr:`_add_task` to add tasks to `c`.""" if self._add_tasks is None: raise NotImplementedError diff --git a/genno/core/quantity.py b/genno/core/quantity.py index fba183c6..eab65788 100644 --- a/genno/core/quantity.py +++ b/genno/core/quantity.py @@ -16,7 +16,7 @@ class Quantity(DataArrayLike["Quantity"]): """A sparse data structure that behaves like :class:`xarray.DataArray`. - Depending on the value of :data:`CLASS`, Quantity is either :class:`.AttrSeries` or + Depending on the value of :data:`.CLASS`, Quantity is either :class:`.AttrSeries` or :class:`.SparseDataArray`. """ diff --git a/genno/core/sparsedataarray.py b/genno/core/sparsedataarray.py index 0050dd44..24658dab 100644 --- a/genno/core/sparsedataarray.py +++ b/genno/core/sparsedataarray.py @@ -82,7 +82,7 @@ def COO_data(self): @property def dense(self): - """Return a copy with dense (:class:`.ndarray`) data.""" + """Return a copy with dense (:class:`numpy.ndarray`) data.""" try: # Use existing method xr.Variable._to_dense() return self.da._replace(variable=self.da.variable._to_dense()) @@ -92,7 +92,7 @@ def dense(self): @property def dense_super(self): - """Return a proxy to a :class:`.ndarray`-backed :class:`.DataArray`.""" + """Return a proxy to a :class:`numpy.ndarray`-backed :class:`xarray.DataArray`.""" return super(SparseDataArray, self.dense) @@ -118,10 +118,11 @@ class SparseDataArray(OverrideItem, xr.DataArray, Quantity): """:class:`~xarray.DataArray` with sparse data. SparseDataArray uses :class:`sparse.COO` for storage with :data:`numpy.nan` - as its :attr:`sparse.COO.fill_value`. Some methods of :class:`~xarray.DataArray` - are overridden to ensure data is in sparse, or dense, format as necessary, to - provide expected functionality not currently supported by :mod:`sparse`, and to - avoid exhausting memory for some operations that require dense data. + as its :attr:`sparse.SparseArray.fill_value`. Some methods of + :class:`~xarray.DataArray` are overridden to ensure data is in sparse, or dense, + format as necessary, to provide expected functionality not currently supported by + :mod:`sparse`, and to avoid exhausting memory for some operations that require dense + data. """ __slots__: Tuple[str, ...] = tuple() @@ -245,7 +246,7 @@ def to_dataframe( name: Optional[Hashable] = None, dim_order: Optional[Sequence[Hashable]] = None, ) -> pd.DataFrame: - """Convert this array and its coords into a :class:`~xarray.DataFrame`. + """Convert this array and its coords into a :class:`pandas.DataFrame`. Overrides :meth:`~xarray.DataArray.to_dataframe`. """ diff --git a/genno/operator.py b/genno/operator.py index 2a9ae8f6..5cdb0974 100644 --- a/genno/operator.py +++ b/genno/operator.py @@ -95,15 +95,15 @@ def add_binop(func, c: "Computer", key, *quantities, **kwargs) -> Key: Parameters ---------- - key : str or Key + key : str or .Key Key or name of the new quantity. If a Key, any dimensions are ignored; the dimensions of the result are the union of the dimensions of `quantities`. - sums : bool, *optional* + sums : bool, optional If :obj:`True`, all partial sums of the new quantity are also added. Returns ------- - Key + .Key The full key of the new quantity. Example @@ -146,7 +146,7 @@ def add(*quantities: Quantity, fill_value: float = 0.0) -> Quantity: Returns ------- - Quantity + .Quantity Units are the same as the first of `quantities`. See also @@ -329,8 +329,8 @@ def broadcast_map( """Broadcast `quantity` using a `map`. The `map` must be a 2-dimensional Quantity with dimensions (``d1``, ``d2``), such as - returned by :func:`map_as_qty`. `quantity` must also have a dimension ``d1``. - Typically ``len(d2) > len(d1)``. + returned by :func:`ixmp.report.operator.map_as_qty`. `quantity` must also have a + dimension ``d1``. Typically ``len(d2) > len(d1)``. `quantity` is 'broadcast' by multiplying it with `map`, and then summing on the common dimension ``d1``. The result has the dimensions of `quantity`, but with @@ -338,9 +338,10 @@ def broadcast_map( Parameters ---------- - rename : dict (str -> str), *optional* - Dimensions to rename on the result. - strict : bool, *optional* + rename : dict, optional + Dimensions to rename on the result; mapping from original dimension + (:class:`str`) to target name (:class:`str`). + strict : bool, optional Require that each element of ``d2`` is mapped from exactly 1 element of ``d1``. """ if strict and int(map.sum().item()) != len(map.coords[map.dims[1]]): @@ -354,11 +355,11 @@ def combine( select: Optional[List[Mapping]] = None, weights: Optional[List[float]] = None, ) -> Quantity: # noqa: F811 - """Sum distinct *quantities* by *weights*. + """Sum distinct `quantities` by `weights`. Parameters ---------- - *quantities : Quantity + *quantities : .Quantity The quantities to be added. select : list of dict Elements to be selected from each quantity. Must have the same number of @@ -370,7 +371,7 @@ def combine( Raises ------ ValueError - If the *quantities* have mismatched units. + If the `quantities` have mismatched units. """ # Handle arguments if select is None: @@ -495,8 +496,8 @@ def div(numerator: Union[Quantity, float], denominator: Quantity) -> Quantity: Parameters ---------- - numerator : Quantity - denominator : Quantity + numerator : .Quantity + denominator : .Quantity See also -------- @@ -505,7 +506,7 @@ def div(numerator: Union[Quantity, float], denominator: Quantity) -> Quantity: return numerator / denominator -#: Alias of :func:`div`, for backwards compatibility. +#: Alias of :func:`~genno.operator.div`, for backwards compatibility. #: #: .. note:: This may be deprecated and possibly removed in a future version. ratio = div @@ -605,7 +606,7 @@ def load_file( units: Optional[UnitLike] = None, name: Optional[str] = None, ) -> Any: - """Read the file at `path` and return its contents as a :class:`.Quantity`. + """Read the file at `path` and return its contents as a :class:`~genno.Quantity`. Some file formats are automatically converted into objects for direct use in genno computations: @@ -624,7 +625,7 @@ def load_file( ---------- path : pathlib.Path Path to the file to read. - dims : collections.abc.Collection or collections.abc.Mapping, *optional* + dims : collections.abc.Collection or collections.abc.Mapping, optional If a collection of names, other columns besides these and 'value' are discarded. If a mapping, the keys are the column labels in `path`, and the values are the target dimension names. @@ -660,7 +661,7 @@ def add_load_file(func, c: "Computer", path, key=None, **kwargs): ---------- path : os.PathLike Path to the file, e.g. '/path/to/foo.ext'. - key : str or Key, *optional* + key : str or .Key, optional Key for the quantity read from the file. Other parameters @@ -767,7 +768,7 @@ def mul(*quantities: Quantity) -> Quantity: return reduce(operator.mul, quantities) -#: Alias of :func:`mul`, for backwards compatibility. +#: Alias of :func:`~genno.operator.mul`, for backwards compatibility. #: #: .. note:: This may be deprecated and possibly removed in a future version. product = mul @@ -778,7 +779,7 @@ def pow(a: Quantity, b: Union[Quantity, int]) -> Quantity: Returns ------- - Quantity + .Quantity If `b` is :class:`int` or a Quantity with all :class:`int` values that are equal to one another, then the quantity has the units of `a` raised to this power; for example, "kg²" → "kg⁴" if `b` is 2. In other cases, there are no meaningful @@ -894,13 +895,17 @@ def select( Parameters ---------- - indexers : dict (str -> xarray.DataArray or list of str) - Elements to be selected from `qty`. Mapping from dimension names to coords along - the respective dimension of `qty`, or to xarray-style indexers. Values not - appearing in the dimension coords are silently ignored. - inverse : bool, *optional* + indexers : dict + Elements to be selected from `qty`. Mapping from dimension names (:class:`str`) + to either: + + - :class:`list` of `str`: coords along the respective dimension of `qty`, or + - :class:`xarray.DataArray`: xarray-style indexers. + + Values not appearing in the dimension coords are silently ignored. + inverse : bool, optional If :obj:`True`, *remove* the items in indexers instead of keeping them. - drop : bool, *optional* + drop : bool, optional If :obj:`True`, drop dimensions that are indexed by a scalar value (for instance, :py:`"foo"` or :py:`999`) in `indexers`. Note that dimensions indexed by a length-1 list of labels (for instance :py:`["foo"]`) are not dropped; this @@ -958,10 +963,10 @@ def sum( Parameters ---------- - weights : .Quantity, *optional* + weights : .Quantity, optional If `dimensions` is given, `weights` must have at least these dimensions. Otherwise, any dimensions are valid. - dimensions : list of str, *optional* + dimensions : list of str, optional If not provided, sum over all dimensions. If provided, sum over these dimensions. """ diff --git a/genno/testing/__init__.py b/genno/testing/__init__.py index 553effe1..7e015c5c 100644 --- a/genno/testing/__init__.py +++ b/genno/testing/__init__.py @@ -60,7 +60,7 @@ def add_large_data(c: Computer, num_params, N_dims=6, N_data=0): The result is a matrix wherein the Cartesian product of all the keys is very large— about 2e17 elements for N_dim = 6—but the contents are very sparse. This can be handled by :class:`.SparseDataArray`, but not by :class:`xarray.DataArray` backed - by :class:`np.array`. + by :class:`numpy.ndarray`. """ def _fib(): @@ -234,7 +234,7 @@ def assert_logs(caplog, message_or_messages=None, at_level=None): The pytest caplog fixture. message_or_messages : str or list of str String(s) that must appear in log messages. - at_level : int, *optional* + at_level : int, optional Messages must appear on 'genno' or a sub-logger with at least this level. """ __tracebackhide__ = True @@ -289,12 +289,12 @@ def assert_qty_equal( Parameters ---------- - check_type : bool, *optional* + check_type : bool, optional Assert that `a` and `b` are both :class:`.Quantity` instances. If :obj:`False`, the arguments are converted to Quantity. - check_attrs : bool, *optional* + check_attrs : bool, optional Also assert that check that attributes are identical. - ignore_extra_coords : bool, *optional* + ignore_extra_coords : bool, optional Ignore extra coords that are not dimensions. Only meaningful when Quantity is :class:`.SparseDataArray`. """ @@ -345,12 +345,12 @@ def assert_qty_allclose( Parameters ---------- - check_type : bool, *optional* + check_type : bool, optional Assert that `a` and `b` are both :class:`.Quantity` instances. If :obj:`False`, the arguments are converted to Quantity. - check_attrs : bool, *optional* + check_attrs : bool, optional Also assert that check that attributes are identical. - ignore_extra_coords : bool, *optional* + ignore_extra_coords : bool, optional Ignore extra coords that are not dimensions. Only meaningful when Quantity is :class:`.SparseDataArray`. """ @@ -392,19 +392,20 @@ def assert_units(qty: Quantity, exp: str) -> None: ).dimensionless, f"Units '{qty.units:~}'; expected {repr(exp)}" -def random_qty(shape: Dict[str, int], **kwargs): +def random_qty(shape: Dict[str, int], **kwargs) -> Quantity: """Return a Quantity with `shape` and random contents. Parameters ---------- - shape : dict (str -> int) - Mapping from dimension names to lengths along each dimension. + shape : dict + Mapping from dimension names (:class:`str`) to lengths along each dimension + (:class:`int`). **kwargs - Other keyword arguments to :class:`Quantity`. + Other keyword arguments to :class:`.Quantity`. Returns ------- - Quantity + .Quantity Random data with one dimension for each key in `shape`, and coords along those dimensions like "foo1", "foo2", with total length matching the value from `shape`. If `shape` is empty, a scalar (0-dimensional) Quantity. diff --git a/genno/testing/jupyter.py b/genno/testing/jupyter.py index e2f8bd13..0819065c 100644 --- a/genno/testing/jupyter.py +++ b/genno/testing/jupyter.py @@ -16,11 +16,11 @@ def run_notebook(nb_path, tmp_path, env=None, **kwargs): Parameters ---------- - nb_path : path-like + nb_path : os.PathLike The notebook file to execute. - tmp_path : path-like + tmp_path : os.PathLike A directory in which to create temporary output. - env : dict-like, *optional* + env : mapping, optional Execution environment for :mod:`nbclient`. Default: :obj:`os.environ`. kwargs : Keyword arguments for :class:`nbclient.NotebookClient`. Defaults are set for: @@ -102,7 +102,7 @@ def get_cell_output(nb, name_or_index, kind="data"): Parameters ---------- - kind : str, *optional* + kind : str, optional Kind of cell output to retrieve. For 'data', the data in format 'text/plain' is run through :func:`eval`. To retrieve an exception message, use 'evalue'. """ diff --git a/genno/util.py b/genno/util.py index d918aabb..57e8879c 100644 --- a/genno/util.py +++ b/genno/util.py @@ -23,7 +23,7 @@ #: - The '%' symbol cannot be supported by pint, because it is a Python operator; it is #: replaced with “percent”. #: -#: Additional values can be added with :func:`configure`; see :ref:`config-units`. +#: Additional values can be added with :func:`.configure`; see :ref:`config-units`. REPLACE_UNITS = { "%": "percent", } @@ -176,7 +176,7 @@ def free_parameters(func: Callable) -> Mapping: """Retrieve information on the free parameters of `func`. Identical to :py:`inspect.signature(func).parameters`; that is, to - :attr:`inspect.Signature.parameters`. :func:`free_pars` also: + :attr:`inspect.Signature.parameters`. :py:`free_parameters` also: - Handles functions that have been :func:`functools.partial`'d, returning only the parameters that have *not* already been assigned a value by the From 07a8e984acf4bb7d6caf227dd7e2b1606f04c451 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 11:40:07 +0100 Subject: [PATCH 13/18] Address Sphinx nitpicks via message_ix --- genno/core/computer.py | 26 ++++++++++++++++---------- genno/core/key.py | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/genno/core/computer.py b/genno/core/computer.py index dc3b3727..1bfc82ce 100644 --- a/genno/core/computer.py +++ b/genno/core/computer.py @@ -1,12 +1,13 @@ import logging +import types from collections import deque from functools import lru_cache, partial from importlib import import_module from inspect import signature from itertools import compress from pathlib import Path -from types import ModuleType from typing import ( + TYPE_CHECKING, Any, Callable, Hashable, @@ -38,6 +39,11 @@ from .graph import Graph from .key import Key, KeyLike +if TYPE_CHECKING: + import genno.core.graph + import genno.core.key + + log = logging.getLogger(__name__) @@ -51,18 +57,18 @@ class Computer: """ #: A dask-format graph (see :doc:`1 `, :doc:`2 `). - graph: Graph = Graph(config=dict()) + graph: "genno.core.graph.Graph" = Graph(config=dict()) #: The default key to :meth:`.get` with no argument. - default_key: Optional[KeyLike] = None + default_key: Optional["genno.core.key.KeyLike"] = None #: List of modules containing operators. #: #: By default, this includes the :mod:`genno` built-in operators in #: :mod:`genno.operator`. :meth:`require_compat` appends additional modules, - #: for instance :mod:`.compat.pyam.operator`, to this list. User code may also add + #: for instance :mod:`genno.compat.plotnine`, to this list. User code may also add #: modules to this list directly. - modules: MutableSequence[ModuleType] = [] + modules: MutableSequence[types.ModuleType] = [] # Action to take on failed items on add_queue(). This is a stack; the rightmost # element is current; the leftmost is the default. @@ -181,7 +187,7 @@ def _get_operator(self, name: str) -> Optional[Callable]: #: Alias of :meth:`get_operator`. get_comp = get_operator - def require_compat(self, pkg: Union[str, ModuleType]): + def require_compat(self, pkg: Union[str, types.ModuleType]): """Register a module for :meth:`get_operator`. The specified module is appended to :attr:`modules`. @@ -219,7 +225,7 @@ def require_compat(self, pkg: Union[str, ModuleType]): >>> c.require_compat(mod) """ - if isinstance(pkg, ModuleType): + if isinstance(pkg, types.ModuleType): mod = pkg elif "." in pkg: mod = import_module(pkg) @@ -444,11 +450,11 @@ def add_single( Raises ------ - KeyExistsError + ~genno.KeyExistsError If `strict` is :obj:`True` and either (a) `key` already exists; or (b) `sums` is :obj:`True` and the key for one of the partial sums of `key` already exists. - MissingKeyError + ~genno.MissingKeyError If `strict` is :obj:`True` and any key referred to by `computation` does not exist. """ @@ -699,7 +705,7 @@ def check_keys( Raises ------ - MissingKeyError + ~genno.MissingKeyError If `action` is "raise" and 1 or more of `keys` do not appear (either in different dimension order, or full dimensionality) in the :attr:`graph`. """ diff --git a/genno/core/key.py b/genno/core/key.py index e384cfbb..abfb1678 100644 --- a/genno/core/key.py +++ b/genno/core/key.py @@ -271,7 +271,7 @@ def tag(self) -> Optional[str]: @property def sorted(self) -> "Key": - """A version of the Key with its :attr:`dims` :func:`sorted`.""" + """A version of the Key with its :attr:`.dims` :func:`sorted`.""" return Key(self._name, sorted(self._dims), self._tag, _fast=True) def rename(self, name: str) -> "Key": From 642afe77f8a1c73892d907d2455a85debe5c53d5 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 11:42:10 +0100 Subject: [PATCH 14/18] =?UTF-8?q?Fix=20typing=20of=20as=5Fpyam(=E2=80=A6,?= =?UTF-8?q?=20rename=3D=E2=80=A6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- genno/compat/pyam/operator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/genno/compat/pyam/operator.py b/genno/compat/pyam/operator.py index 04ecf93a..83b23e48 100644 --- a/genno/compat/pyam/operator.py +++ b/genno/compat/pyam/operator.py @@ -35,7 +35,7 @@ def as_pyam( scenario, quantity: "Quantity", *, - rename=Mapping[str, str], + rename: Optional[Mapping[str, str]] = None, collapse: Optional[Callable] = None, replace=dict(), drop: Union[Collection[str], str] = "auto", @@ -107,7 +107,7 @@ def as_pyam( model=scenario.model, scenario=scenario.scenario, ) - .rename(columns=rename) + .rename(columns=rename or dict()) .pipe(collapse or util.collapse) .replace(replace, regex=True) .pipe(util.drop, columns=drop) From 5acb6ff7a48472a4b9fa222f5b85ff1af7d8b4bb Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 12:06:32 +0100 Subject: [PATCH 15/18] Add object, str implementations for write_report() Expand tests. --- genno/operator.py | 48 ++++++++++++++++++++++++++----- genno/tests/core/test_computer.py | 2 +- genno/tests/test_operator.py | 26 +++++++++++++++++ 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/genno/operator.py b/genno/operator.py index 5cdb0974..cb7e031d 100644 --- a/genno/operator.py +++ b/genno/operator.py @@ -1015,24 +1015,56 @@ def _format_header_comment(value: str) -> str: @singledispatch def write_report( - quantity: pd.DataFrame, path: Union[str, PathLike], kwargs: Optional[dict] = None + quantity: object, path: Union[str, PathLike], kwargs: Optional[dict] = None ) -> None: """Write a quantity to a file. + :py:`write_report()` is a :func:`~functools.singledispatch` function. This means + that user code can extend this operator to support different types for the + `quantity` argument: + + .. code-block:: python + + import genno.operator + + @genno.operator.write_report.register + def my_writer(qty: MyClass, path, kwargs): + ... # Code to write MyClass to file + Parameters ---------- quantity : + Object to be written. The base implementation supports :class:`.Quantity` and + :class:`pandas.DataFrame`. path : str or pathlib.Path Path to the file to be written. kwargs : - Keyword arguments. For the default implementation, these are passed to - :meth:~pandas.DataFrame.to_csv` or :meth:~pandas.DataFrame.to_excel` (according + Keyword arguments. For the base implementation, these are passed to + :meth:`pandas.DataFrame.to_csv` or :meth:`pandas.DataFrame.to_excel` (according to `path`), except for: - "header_comment": valid only for `path` ending in :file:`.csv`. Multi-line text that is prepended to the file, with comment characters ("# ") before each line. + + Raises + ------ + NotImplementedError + If `quantity` is of a type not supported by the base implementation or any + overloads. """ + raise NotImplementedError(f"Write {type(quantity)} to file") + + +@write_report.register +def _(quantity: str, path: Union[str, PathLike], kwargs: Optional[dict] = None): + Path(path).write_text(quantity) + + +@write_report.register +def _( + quantity: pd.DataFrame, path: Union[str, PathLike], kwargs: Optional[dict] = None +) -> None: path = Path(path) if path.suffix == ".csv": @@ -1044,15 +1076,17 @@ def write_report( quantity.to_csv(f, **kwargs) elif path.suffix == ".xlsx": kwargs = kwargs or dict() - kwargs.setdefault("index", False) kwargs.setdefault("merge_cells", False) + kwargs.setdefault("index", False) quantity.to_excel(path, **kwargs) else: - path.write_text(quantity) # type: ignore + raise NotImplementedError(f"Write pandas.DataFrame to {path.suffix!r}") @write_report.register -def _(quantity: Quantity, path, kwargs=None) -> None: +def _( + quantity: Quantity, path: Union[str, PathLike], kwargs: Optional[dict] = None +) -> None: # Convert the Quantity to a pandas.DataFrame, then write - write_report(quantity.to_dataframe(), path, kwargs) + write_report(quantity.to_dataframe().reset_index(), path, kwargs) diff --git a/genno/tests/core/test_computer.py b/genno/tests/core/test_computer.py index ed10c868..096056a9 100644 --- a/genno/tests/core/test_computer.py +++ b/genno/tests/core/test_computer.py @@ -798,7 +798,7 @@ def test_file_formats(test_data_path, tmp_path): # Write to CSV p3 = tmp_path / "output.csv" - c.write(k, p3, index=True) + c.write(k, p3) # Output is identical to input file, except for order assert sorted(p1.read_text().split("\n")) == sorted(p3.read_text().split("\n")) diff --git a/genno/tests/test_operator.py b/genno/tests/test_operator.py index c55e40c8..acbe2793 100644 --- a/genno/tests/test_operator.py +++ b/genno/tests/test_operator.py @@ -887,3 +887,29 @@ def test_sum(data, dimensions): result = operator.sum(x, dimensions=dimensions) assert result.name == x.name and result.units == x.units # Pass through + + +def test_write_report0(tmp_path, data) -> None: + p = tmp_path.joinpath("foo.txt") + *_, x = data + + # Unsupported type + with pytest.raises(NotImplementedError, match="Write to file"): + operator.write_report(list(), p) + + # Unsupported path suffix + with pytest.raises(NotImplementedError, match="Write pandas.DataFrame to '.bar'"): + operator.write_report(x, tmp_path.joinpath("foo.bar")) + + # Plain text + operator.write_report("Hello, world!", p) + assert "Hello, world!" == p.read_text() + + +def test_write_report1(tmp_path, data) -> None: + p = tmp_path.joinpath("foo.csv") + *_, x = data + + # Header comment is written + operator.write_report(x, p, dict(header_comment="Hello, world!\n")) + assert p.read_text().startswith("# Hello, world!\n#") From 0a911401a6c89ca2142bfc4a48a7683498a13d05 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 12:19:18 +0100 Subject: [PATCH 16/18] Add #108 to doc/whatsnew --- doc/whatsnew.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 3ebf0048..fa8bd15c 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -1,8 +1,12 @@ What's new ********** -.. Next release -.. ============ +Next release +============ + +- :func:`.write_report` and :func:`.concat` are single-dispatch functions for simpler extension in user code (:pull:`108`). +- New argument to :func:`.write_report`: :py:`kwargs`, including "header_comment" to write a header comment at the start of a :file:`.csv` file (:pull:`108`). +- Fix many cross-references in the documentation (:pull:`108`). v1.20.0 (2023-10-28) ==================== From f1e9a26a19053fd609d143f07ac2b840f88f92a3 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 12:27:43 +0100 Subject: [PATCH 17/18] Make _format_header_comment() cross-platform --- genno/operator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/genno/operator.py b/genno/operator.py index cb7e031d..61281f8d 100644 --- a/genno/operator.py +++ b/genno/operator.py @@ -4,6 +4,7 @@ import logging import numbers import operator +import os import re from functools import partial, reduce, singledispatch from itertools import chain @@ -1010,7 +1011,7 @@ def _format_header_comment(value: str) -> str: from textwrap import indent - return indent(value + "\n", "# ", lambda line: True) + return indent(value + os.linesep, "# ", lambda line: True) @singledispatch @@ -1071,8 +1072,8 @@ def _( kwargs = kwargs or dict() kwargs.setdefault("index", False) - with open(path, "w") as f: - f.write(_format_header_comment(kwargs.pop("header_comment", ""))) + with open(path, "wb") as f: + f.write(_format_header_comment(kwargs.pop("header_comment", "")).encode()) quantity.to_csv(f, **kwargs) elif path.suffix == ".xlsx": kwargs = kwargs or dict() From 409f29a5663807d7b33d2e6f39e4d6f5664e8858 Mon Sep 17 00:00:00 2001 From: Paul Natsuo Kishimoto Date: Tue, 28 Nov 2023 12:58:24 +0100 Subject: [PATCH 18/18] Restore coverage in pyam.operator.write_report --- genno/tests/compat/test_pyam.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/genno/tests/compat/test_pyam.py b/genno/tests/compat/test_pyam.py index 9a3b9b35..daf5756f 100644 --- a/genno/tests/compat/test_pyam.py +++ b/genno/tests/compat/test_pyam.py @@ -169,7 +169,7 @@ def add_tm(df, name="Activity"): ) assert_frame_equal(df2[["region", "variable"]], reg_var) - # pyam.operator.write_file() is used, calling pyam.IamDataFrame.to_csv() + # pyam.operator.write_report() is used, calling pyam.IamDataFrame.to_csv() path = tmp_path / "activity.csv" c.write(key2, path) @@ -183,6 +183,10 @@ def add_tm(df, name="Activity"): with pytest.raises(ValueError, match=".csv or .xlsx, not .foo"): c.write(key2, tmp_path / "activity.foo") + # Giving keyword arguments raises exception + with pytest.raises(NotImplementedError): + c.write(key2, tmp_path / "activity.csv", index=False) + # Non-pyam objects are written using base write_file() c.write(ACT, tmp_path / "ACT.csv")