♻️ Maintenance/new settings: settings-library and cleanup (#2736)

.pre-commit-config.yaml

@@ -16,7 +16,6 @@ repos:
     rev: v1.1.0
     hooks:
       - id: pycln
-        args: [--all]
   - repo: https://github.com/PyCQA/isort
     rev: 5.6.4
     hooks:

packages/models-library/src/models_library/services.py

@@ -45,6 +45,9 @@
 FileName = constr(regex=FILENAME_RE)
 GroupId = PositiveInt
 
+ServiceKey = constr(regex=KEY_RE)
+ServiceVersion = constr(regex=VERSION_RE)
+
 
 class ServiceType(str, Enum):
     COMPUTATIONAL = "computational"

packages/pytest-simcore/src/pytest_simcore/helpers/utils_pylint.py

@@ -30,6 +30,8 @@ def assert_pylint_is_passing(pylintrc, package_dir, number_of_jobs: int = AUTODE
 
 
 def assert_no_pdb_in_code(code_dir: Path):
+    # TODO: deprecate since Pylint 2.10 adds 'forgotten-debug-statement'
+    # https://pylint.pycqa.org/en/latest/whatsnew/2.10.html?highlight=forgotten-debug-statement#new-checkers
     for root, dirs, files in os.walk(code_dir):
         for name in files:
             if name.endswith(".py"):

packages/service-library/src/servicelib/aiohttp/application_keys.py

@@ -8,24 +8,24 @@
 
  See https://aiohttp.readthedocs.io/en/stable/web_advanced.html#data-sharing-aka-no-singletons-please
 """
+from typing import Final
 
 # REQUIREMENTS:
 # - guarantees all keys are unique
 # - one place for all common keys
 # - hierarchical classification
-# TODO: should be read-only (frozen?)
 
 #
 # web.Application keys, i.e. app[APP_*_KEY]
 #
+APP_CONFIG_KEY: Final[str] = f"{__name__ }.config"
+APP_JSONSCHEMA_SPECS_KEY: Final[str] = f"{__name__ }.jsonschema_specs"
+APP_OPENAPI_SPECS_KEY: Final[str] = f"{__name__ }.openapi_specs"
+APP_SETTINGS_KEY: Final[str] = f"{__name__ }.settings"
 
-APP_CONFIG_KEY = f"{__name__ }.config"
-APP_OPENAPI_SPECS_KEY = f"{__name__ }.openapi_specs"
-APP_JSONSCHEMA_SPECS_KEY = f"{__name__ }.jsonschema_specs"
+APP_DB_ENGINE_KEY: Final[str] = f"{__name__ }.db_engine"
 
-APP_DB_ENGINE_KEY = f"{__name__ }.db_engine"
-
-APP_CLIENT_SESSION_KEY = f"{__name__ }.session"
+APP_CLIENT_SESSION_KEY: Final[str] = f"{__name__ }.session"
 
 #
 # web.Response keys, i.e. app[RSP_*_KEY]

packages/service-library/src/servicelib/aiohttp/web_exceptions_extension.py

@@ -0,0 +1,6 @@
+from aiohttp.web_exceptions import HTTPClientError
+
+
+class HTTPLocked(HTTPClientError):
+    # pylint: disable=too-many-ancestors
+    status_code = 423

packages/settings-library/src/settings_library/_constants.py

@@ -0,0 +1,6 @@
+# Units
+
+GB = 1024 ** 3
+
+# Formatting
+HEADER_STR = "{:-^50}"

packages/settings-library/src/settings_library/basic_types.py

@@ -1,9 +1,18 @@
+#
+# NOTE: This files copies some of the types from models_library.basic_types
+#       This is a minor evil to avoid the maintenance burden that creates
+#       an extra dependency to a larger models_library (intra-repo library)
+
 from enum import Enum
 
-from pydantic import conint
+from pydantic.types import conint, constr
 
+# port number range
 PortInt = conint(gt=0, lt=65535)
 
+# e.g. 'v5'
+VersionTag = constr(regex=r"^v\d$")
+
 
 class LogLevel(str, Enum):
     DEBUG = "DEBUG"

packages/settings-library/src/settings_library/comp_services.py

@@ -2,7 +2,7 @@
 from pydantic.tools import parse_raw_as
 from settings_library.base import BaseCustomSettings
 
-from .constants import GB
+from ._constants import GB
 
 _DEFAULT_MAX_NANO_CPUS_VALUE = 1 * pow(10, 9)
 _DEFAULT_MAX_MEMORY_VALUE = 2 * GB

packages/settings-library/src/settings_library/email.py

@@ -0,0 +1,18 @@
+from pydantic.fields import Field
+from pydantic.types import SecretStr
+
+from .base import BaseCustomSettings
+from .basic_types import PortInt
+
+
+class SMTPSettings(BaseCustomSettings):
+    """Simple Mail Transfer Protocol"""
+
+    SMTP_SENDER: str = "@".join(["O2SPARC support <support", "osparc.io>"])
+
+    SMTP_HOST: str
+    SMTP_PORT: PortInt
+
+    SMTP_TLS_ENABLED: bool = Field(description="Enables Secure Mode")
+    SMTP_USERNAME: str
+    SMTP_PASSWORD: SecretStr

packages/settings-library/src/settings_library/prometheus.py

@@ -0,0 +1,22 @@
+from functools import cached_property
+
+from pydantic.networks import HttpUrl
+from settings_library.base import BaseCustomSettings
+from settings_library.utils_service import MixinServiceSettings
+
+from .basic_types import PortInt, VersionTag
+
+
+class PrometheusSettings(BaseCustomSettings, MixinServiceSettings):
+    PROMETHEUS_HOST: str = "prometheus"
+    PROMETHEUS_PORT: PortInt = 9090
+    PROMETHEUS_VTAG: VersionTag = "v1"
+
+    @cached_property
+    def base_url(self) -> str:
+        return HttpUrl.build(
+            scheme="http",
+            host=self.PROMETHEUS_HOST,
+            port=f"{self.PROMETHEUS_PORT}",
+            path=f"/api/{self.PROMETHEUS_VTAG}/query",
+        )

packages/settings-library/src/settings_library/cli_utils.py → packages/settings-library/src/settings_library/utils_cli.py

@@ -7,10 +7,9 @@
 from pydantic import ValidationError
 from pydantic.env_settings import BaseSettings
 
+from ._constants import HEADER_STR
 from .base import BaseCustomSettings
 
-HEADER = "{:-^50}"
-
 
 def print_as_envfile(settings_obj, *, compact, verbose):
     for name in settings_obj.__fields__:
@@ -71,17 +70,17 @@ def settings(
                 "Invalid application settings. Typically an environment variable is missing or mistyped :\n%s",
                 "\n".join(
                     [
-                        HEADER.format("detail"),
+                        HEADER_STR.format("detail"),
                         str(err),
-                        HEADER.format("environment variables"),
+                        HEADER_STR.format("environment variables"),
                         pformat(
                             {
                                 k: v
                                 for k, v in dict(os.environ).items()
                                 if k.upper() == k
                             }
                         ),
-                        HEADER.format("json-schema"),
+                        HEADER_STR.format("json-schema"),
                         settings_schema,
                     ]
                 ),

packages/settings-library/src/settings_library/utils_service.py

@@ -0,0 +1,72 @@
+""" Helpers to build settings for services with http API
+
+
+"""
+
+
+from pydantic.networks import AnyUrl
+
+from .basic_types import PortInt
+
+DEFAULT_AIOHTTP_PORT: PortInt = 8000
+DEFAULT_FASTAPI_PORT: PortInt = 8080
+
+
+class MixinServiceSettings:
+    """Mixin with common helpers based on validated fields with canonical name
+
+    Example:
+       - Subclass should define host, port and vtag fields as
+
+        class MyServiceSettings(BaseCustomSettings, MixinServiceSettings):
+            {prefix}_HOST: str
+            {prefix}_PORT: PortInt
+            {prefix}_VTAG: VersionTag  [Optional]
+
+            # optional
+            {prefix}_SCHEME: str (urls default to http)
+            {prefix}_USER: str
+            {prefix}_PASSWORD: SecretStr
+    """
+
+    #
+    # URL conventions (based on https://yarl.readthedocs.io/en/latest/api.html)
+    #
+    #     http://user:[email protected]:8042/v0/resource?name=ferret#nose
+    #     \__/   \__/ \__/ \_________/ \__/\__________/ \_________/ \__/
+    #     |      |    |        |       |      |           |        |
+    #     scheme  user password host    port   path       query   fragment
+    #
+    # origin    -> http://example.com
+    # api_base  -> http://example.com:8042/v0
+    #
+
+    def _build_api_base_url(self, *, prefix: str) -> str:
+        assert prefix  # nosec
+        prefix = prefix.upper()
+        password = getattr(self, f"{prefix}_PASSWORD")
+        vtag = getattr(self, f"{prefix}_VTAG", None)
+        return AnyUrl.build(
+            scheme=getattr(self, f"{prefix}_SCHEME", "http"),
+            user=getattr(self, f"{prefix}_USER", None),
+            password=password.get_secret_value() if password is not None else None,
+            host=getattr(self, f"{prefix}_HOST"),
+            port=f"{getattr(self, f'{prefix}_PORT')}",
+            path=f"/{vtag}" if vtag is not None else None,
+            query=None,
+            fragment=None,
+        )
+
+    def _build_origin_url(self, *, prefix: str) -> str:
+        assert prefix  # nosec
+        prefix = prefix.upper()
+        return AnyUrl.build(
+            scheme=getattr(self, f"{prefix}_SCHEME", "http"),
+            user=None,
+            password=None,
+            host=getattr(self, f"{prefix}_HOST"),
+            port=None,
+            path=None,
+            query=None,
+            fragment=None,
+        )

packages/settings-library/tests/test_cli_utils.py → packages/settings-library/tests/test_utils_cli.py

@@ -12,7 +12,7 @@
 from dotenv import dotenv_values
 from pydantic import ValidationError
 from settings_library.base import BaseCustomSettings
-from settings_library.cli_utils import create_settings_command
+from settings_library.utils_cli import create_settings_command
 from typer.testing import CliRunner
 
 log = logging.getLogger(__name__)

packages/settings-library/tests/test_logging_utils.py → packages/settings-library/tests/test_utils_logging.py

@@ -4,7 +4,7 @@
 from pydantic import Field, validator
 from settings_library.base import BaseCustomSettings
 from settings_library.basic_types import BootMode
-from settings_library.logging_utils import MixinLoggingSettings
+from settings_library.utils_logging import MixinLoggingSettings
 
 
 def test_mixin_logging(monkeypatch):

packages/settings-library/tests/test_utils_service.py

@@ -0,0 +1,50 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+# pylint: disable=unused-variable
+
+from functools import cached_property
+from typing import Optional
+
+from pydantic.types import SecretStr
+from settings_library.base import BaseCustomSettings
+from settings_library.basic_types import PortInt, VersionTag
+from settings_library.utils_service import DEFAULT_AIOHTTP_PORT, MixinServiceSettings
+
+
+def test_mixing_service_settings_usage(monkeypatch):
+    # this test provides an example of usage
+    class MySettings(BaseCustomSettings, MixinServiceSettings):
+        MY_HOST: str = "example.com"
+        MY_PORT: PortInt = DEFAULT_AIOHTTP_PORT
+        MY_VTAG: Optional[VersionTag] = None
+
+        # optional
+        MY_USER: Optional[str]
+        MY_PASSWORD: Optional[SecretStr]
+
+        @cached_property
+        def api_base_url(self) -> str:
+            return self._build_api_base_url(prefix="MY")
+
+        @cached_property
+        def origin_url(self) -> str:
+            return self._build_origin_url(prefix="MY")
+
+    settings = MySettings.create_from_envs()
+    assert settings.api_base_url == "http://example.com:8000"
+    assert settings.origin_url == "http://example.com"
+
+    # -----------
+    monkeypatch.setenv("MY_VTAG", "v9")
+
+    settings = MySettings.create_from_envs()
+    assert settings.api_base_url == "http://example.com:8000/v9"
+    assert settings.origin_url == "http://example.com"
+
+    # -----------
+    monkeypatch.setenv("MY_USER", "me")
+    monkeypatch.setenv("MY_PASSWORD", "secret")
+
+    settings = MySettings.create_from_envs()
+    assert settings.api_base_url == "http://me:[email protected]:8000/v9"
+    assert settings.origin_url == "http://example.com"

services/api-server/src/simcore_service_api_server/cli.py

@@ -1,7 +1,7 @@
 import logging
 
 import typer
-from settings_library.cli_utils import create_settings_command
+from settings_library.utils_cli import create_settings_command
 
 from ._meta import PROJECT_NAME
 from .core.settings import AppSettings

services/api-server/src/simcore_service_api_server/core/settings.py

@@ -6,9 +6,9 @@
 from pydantic.class_validators import validator
 from pydantic.networks import HttpUrl
 from settings_library.base import BaseCustomSettings
-from settings_library.logging_utils import MixinLoggingSettings
 from settings_library.postgres import PostgresSettings
 from settings_library.tracing import TracingSettings
+from settings_library.utils_logging import MixinLoggingSettings
 
 # SERVICES CLIENTS --------------------------------------------
 

services/catalog/src/simcore_service_catalog/core/settings.py

@@ -6,9 +6,9 @@
 from pydantic import Field, PositiveInt
 from settings_library.base import BaseCustomSettings
 from settings_library.http_client_request import ClientRequestSettings
-from settings_library.logging_utils import MixinLoggingSettings
 from settings_library.postgres import PostgresSettings
 from settings_library.tracing import TracingSettings
+from settings_library.utils_logging import MixinLoggingSettings
 
 logger = logging.getLogger(__name__)
 

services/dask-sidecar/src/simcore_service_dask_sidecar/settings.py

@@ -4,7 +4,7 @@
 from models_library.basic_types import LogLevel
 from pydantic import Field, validator
 from settings_library.base import BaseCustomSettings
-from settings_library.logging_utils import MixinLoggingSettings
+from settings_library.utils_logging import MixinLoggingSettings
 
 
 class Settings(BaseCustomSettings, MixinLoggingSettings):

services/datcore-adapter/src/simcore_service_datcore_adapter/core/settings.py

@@ -4,8 +4,8 @@
 from pydantic import Field
 from pydantic.networks import AnyUrl
 from settings_library.base import BaseCustomSettings
-from settings_library.logging_utils import MixinLoggingSettings
 from settings_library.tracing import TracingSettings
+from settings_library.utils_logging import MixinLoggingSettings
 
 
 class PennsieveSettings(BaseCustomSettings):

services/director-v2/src/simcore_service_director_v2/core/settings.py

@@ -17,10 +17,10 @@
 from settings_library.base import BaseCustomSettings
 from settings_library.docker_registry import RegistrySettings
 from settings_library.http_client_request import ClientRequestSettings
-from settings_library.logging_utils import MixinLoggingSettings
 from settings_library.postgres import PostgresSettings
 from settings_library.rabbit import RabbitSettings
 from settings_library.tracing import TracingSettings
+from settings_library.utils_logging import MixinLoggingSettings
 
 from ..meta import API_VTAG
 from ..models.schemas.constants import DYNAMIC_SIDECAR_DOCKER_IMAGE_RE, ClusterID

services/director-v2/src/simcore_service_director_v2/utils/client_decorators.py

@@ -71,10 +71,10 @@ async def wrapper_func(*args, **kwargs) -> httpx.Response:
 
                 if httpx.codes.is_server_error(resp.status_code):  # i.e. 5XX error
                     logger.error(
-                        "%s service error %d [%s]: %s",
+                        "%s service error %s [%s]: %s",
                         service_name,
                         resp.reason_phrase,
-                        resp.status_code,
+                        f"{resp.status_code=}",
                         resp.text,
                     )
                     raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE)