diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62a9a76..9148b7c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: - --force-single-line-imports - --profile black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.2 + rev: v0.8.1 hooks: - id: ruff args: @@ -64,7 +64,7 @@ repos: hooks: - id: pyproject-fmt - repo: https://github.com/jshwi/docsig # Check docstrings against function sig - rev: v0.64.0 + rev: v0.65.0 hooks: - id: docsig args: @@ -74,7 +74,6 @@ repos: - --check-protected # Check protected methods - --check-class # Check class docstrings - --disable=E113 # Disable empty docstrings - - --summary # Print a summary ci: autoupdate_schedule: monthly ci: diff --git a/src/anemoi/utils/commands/requests.py b/src/anemoi/utils/commands/requests.py new file mode 100644 index 0000000..5f3cd18 --- /dev/null +++ b/src/anemoi/utils/commands/requests.py @@ -0,0 +1,44 @@ +# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +import json + +from anemoi.utils.mars.requests import print_request + +from . import Command + + +class Requests(Command): + """Convert a JSON requests file to MARS format.""" + + def add_arguments(self, command_parser): + command_parser.add_argument("input") + command_parser.add_argument("output") + command_parser.add_argument("--verb", default="retrieve") + command_parser.add_argument("--only-one-field", action="store_true") + + def run(self, args): + with open(args.input) as f: + requests = json.load(f) + + if args.only_one_field: + for r in requests: + for key in ( + "grid", + "area", + ): + r.pop(key, None) + for k, v in list(r.items()): + if isinstance(v, list): + r[k] = v[-1] + + with open(args.output, "w") as f: + for r in requests: + print_request(args.verb, r, file=f) + + +command = Requests diff --git a/src/anemoi/utils/config.py b/src/anemoi/utils/config.py index 64da8d0..c547672 100644 --- a/src/anemoi/utils/config.py +++ b/src/anemoi/utils/config.py @@ -50,14 +50,14 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) for k, v in self.items(): - if isinstance(v, dict): + if isinstance(v, dict) or is_omegaconf_dict(v): self[k] = DotDict(v) - if isinstance(v, list): - self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v] + if isinstance(v, list) or is_omegaconf_list(v): + self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v] if isinstance(v, tuple): - self[k] = [DotDict(i) if isinstance(i, dict) else i for i in v] + self[k] = [DotDict(i) if isinstance(i, dict) or is_omegaconf_dict(i) else i for i in v] @classmethod def from_file(cls, path: str): @@ -106,6 +106,24 @@ def __repr__(self) -> str: return f"DotDict({super().__repr__()})" +def is_omegaconf_dict(value) -> bool: + try: + from omegaconf import DictConfig + + return isinstance(value, DictConfig) + except ImportError: + return False + + +def is_omegaconf_list(value) -> bool: + try: + from omegaconf import ListConfig + + return isinstance(value, ListConfig) + except ImportError: + return False + + CONFIG = {} CHECKED = {} CONFIG_LOCK = threading.RLock() @@ -376,3 +394,11 @@ def find(metadata, what, result=None, *, select: callable = None): find(v, what, result) return result + + +def merge_configs(*configs): + result = {} + for config in configs: + _merge_dicts(result, config) + + return result diff --git a/src/anemoi/utils/logs.py b/src/anemoi/utils/logs.py new file mode 100644 index 0000000..47d4798 --- /dev/null +++ b/src/anemoi/utils/logs.py @@ -0,0 +1,40 @@ +# (C) Copyright 2024 Anemoi contributors. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + + +"""Logging utilities.""" + +import logging +import threading + +thread_local = threading.local() + + +LOGGER = logging.getLogger(__name__) + + +def set_logging_name(name): + thread_local.logging_name = name + + +class ThreadCustomFormatter(logging.Formatter): + def format(self, record): + record.logging_name = thread_local.logging_name + return super().format(record) + + +def enable_logging_name(name="main"): + thread_local.logging_name = name + + formatter = ThreadCustomFormatter("%(asctime)s - %(logging_name)s - %(levelname)s - %(message)s") + + logger = logging.getLogger() + + for handler in logger.handlers: + handler.setFormatter(formatter) diff --git a/src/anemoi/utils/mars/requests.py b/src/anemoi/utils/mars/requests.py new file mode 100644 index 0000000..66bd568 --- /dev/null +++ b/src/anemoi/utils/mars/requests.py @@ -0,0 +1,22 @@ +# (C) Copyright 2024 European Centre for Medium-Range Weather Forecasts. +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +import sys + + +def print_request(verb, request, file=sys.stdout): + r = [verb] + for k, v in request.items(): + if not isinstance(v, (list, tuple, set)): + v = [v] + v = [str(_) for _ in v] + v = "/".join(v) + r.append(f"{k}={v}") + + r = ",\n ".join(r) + print(r, file=file) + print(file=file)