♻️ Maintenance/new settings (2nd round)

.pre-commit-config.yaml

@@ -16,6 +16,7 @@ repos:
     rev: v1.1.0
     hooks:
       - id: pycln
+        args: [--all]
   - repo: https://github.com/PyCQA/isort
     rev: 5.6.4
     hooks:

packages/pytest-simcore/src/pytest_simcore/helpers/utils_projects.py

@@ -9,11 +9,11 @@
 from typing import Dict, Optional
 
 from aiohttp import web
+from simcore_service_webserver._resources import resources
 from simcore_service_webserver.projects.projects_db import (
     APP_PROJECT_DBAPI,
     DB_EXCLUSIVE_COLUMNS,
 )
-from simcore_service_webserver.resources import resources
 from simcore_service_webserver.utils import now_str
 
 fake_template_resources = [

services/web/server/src/simcore_service_webserver/activity/_schema.py

@@ -0,0 +1,20 @@
+""" Activity manager configuration
+    - config-file schema
+    - prometheus endpoint information
+"""
+
+import trafaret as T
+
+CONFIG_SECTION_NAME = "activity"
+
+
+schema = T.Dict(
+    {
+        T.Key("enabled", default=True, optional=True): T.Bool(),
+        T.Key(
+            "prometheus_host", default="http://prometheus", optional=False
+        ): T.String(),
+        T.Key("prometheus_port", default=9090, optional=False): T.ToInt(),
+        T.Key("prometheus_api_version", default="v1", optional=False): T.String(),
+    }
+)

services/web/server/src/simcore_service_webserver/activity/config.py

@@ -2,41 +2,5 @@
     - config-file schema
     - prometheus endpoint information
 """
-from typing import Dict
-
-import trafaret as T
-from aiohttp.web import Application
-from models_library.basic_types import PortInt, VersionTag
-from pydantic import BaseSettings
-from servicelib.aiohttp.application_keys import APP_CONFIG_KEY
 
 CONFIG_SECTION_NAME = "activity"
-
-
-schema = T.Dict(
-    {
-        T.Key("enabled", default=True, optional=True): T.Bool(),
-        T.Key(
-            "prometheus_host", default="http://prometheus", optional=False
-        ): T.String(),
-        T.Key("prometheus_port", default=9090, optional=False): T.ToInt(),
-        T.Key("prometheus_api_version", default="v1", optional=False): T.String(),
-    }
-)
-
-
-class ActivitySettings(BaseSettings):
-    enabled: bool = True
-    prometheus_host: str = "prometheus"
-    prometheus_port: PortInt = 9090
-    prometheus_api_version: VersionTag = "v1"
-
-    class Config:
-        case_sensitive = False
-        env_prefix = "WEBSERVER_"
-
-
-def assert_valid_config(app: Application) -> Dict:
-    cfg = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
-    _settings = ActivitySettings(**cfg)
-    return cfg

services/web/server/src/simcore_service_webserver/activity/module_setup.py

@@ -9,7 +9,7 @@
 
 from .._constants import APP_OPENAPI_SPECS_KEY
 from . import handlers
-from .config import assert_valid_config
+from .settings import assert_valid_config
 
 logger = logging.getLogger(__name__)
 

services/web/server/src/simcore_service_webserver/activity/settings.py

@@ -0,0 +1,29 @@
+""" Activity manager configuration
+    - config-file schema
+    - prometheus endpoint information
+"""
+from typing import Dict
+
+from aiohttp.web import Application
+from models_library.basic_types import PortInt, VersionTag
+from pydantic import BaseSettings
+from servicelib.aiohttp.application_keys import APP_CONFIG_KEY
+
+from .config import CONFIG_SECTION_NAME
+
+
+class ActivitySettings(BaseSettings):
+    enabled: bool = True
+    prometheus_host: str = "prometheus"
+    prometheus_port: PortInt = 9090
+    prometheus_api_version: VersionTag = "v1"
+
+    class Config:
+        case_sensitive = False
+        env_prefix = "WEBSERVER_"
+
+
+def assert_valid_config(app: Application) -> Dict:
+    cfg = app[APP_CONFIG_KEY][CONFIG_SECTION_NAME]
+    _settings = ActivitySettings(**cfg)
+    return cfg

services/web/server/src/simcore_service_webserver/application__schema.py

@@ -0,0 +1,125 @@
+""" app's configuration
+
+    This module loads the schema defined by every subsystem and injects it in the
+    application's configuration scheams
+
+    It was designed in a similar fashion to the setup protocol of the application
+    where every subsystem is imported and queried in a specific order. The application
+    depends on the subsystem and not the other way around.
+
+    The app configuration is created before the application instance exists.
+
+"""
+import logging
+from pathlib import Path
+from typing import Dict
+
+import trafaret as T
+from trafaret_config.simple import read_and_validate
+
+from . import (
+    catalog__schema,
+    db__schema,
+    email__schema,
+    rest__schema,
+    session__schema,
+    storage__schema,
+    tracing,
+)
+from ._resources import resources
+from .activity import _schema as activity__schema
+from .director import _schema as director__schema
+from .login import _schema as login__schema
+from .projects import _schema as projects__schema
+from .resource_manager import _schema as resource_manager__schema
+from .socketio import _schema as socketio__schema
+
+log = logging.getLogger(__name__)
+
+CLI_DEFAULT_CONFIGFILE = "server-defaults.yaml"
+assert resources.exists("config/" + CLI_DEFAULT_CONFIGFILE)  # nosec
+
+
+def addon_section(name: str, optional: bool = False) -> T.Key:
+    if optional:
+        return T.Key(name, default=dict(enabled=True), optional=optional)
+    return T.Key(name)
+
+
+def minimal_addon_schema() -> T.Dict:
+    return T.Dict({T.Key("enabled", default=True, optional=True): T.Bool()})
+
+
+def create_schema() -> T.Dict:
+    """
+    Build schema for the configuration's file
+    by aggregating all the subsystem configurations
+    """
+    # pylint: disable=protected-access
+    schema = T.Dict(
+        {
+            "version": T.String(),
+            "main": T.Dict(
+                {
+                    "host": T.IP,
+                    "port": T.ToInt(),
+                    "log_level": T.Enum(*logging._nameToLevel.keys()),
+                    "testing": T.Bool(),
+                    T.Key("studies_access_enabled", default=False): T.Or(
+                        T.Bool(), T.ToInt
+                    ),
+                }
+            ),
+            addon_section(tracing.CONFIG_SECTION_NAME, optional=True): tracing.schema,
+            db__schema.CONFIG_SECTION_NAME: db__schema.schema,
+            director__schema.CONFIG_SECTION_NAME: director__schema.schema,
+            rest__schema.CONFIG_SECTION_NAME: rest__schema.schema,
+            projects__schema.CONFIG_SECTION_NAME: projects__schema.schema,
+            email__schema.CONFIG_SECTION_NAME: email__schema.schema,
+            storage__schema.CONFIG_SECTION_NAME: storage__schema.schema,
+            addon_section(
+                login__schema.CONFIG_SECTION_NAME, optional=True
+            ): login__schema.schema,
+            addon_section(
+                socketio__schema.CONFIG_SECTION_NAME, optional=True
+            ): socketio__schema.schema,
+            session__schema.CONFIG_SECTION_NAME: session__schema.schema,
+            activity__schema.CONFIG_SECTION_NAME: activity__schema.schema,
+            resource_manager__schema.CONFIG_SECTION_NAME: resource_manager__schema.schema,
+            # BELOW HERE minimal sections until more options are needed
+            addon_section("catalog", optional=True): catalog__schema.schema,
+            addon_section("clusters", optional=True): minimal_addon_schema(),
+            addon_section("computation", optional=True): minimal_addon_schema(),
+            addon_section("diagnostics", optional=True): minimal_addon_schema(),
+            addon_section("director-v2", optional=True): minimal_addon_schema(),
+            addon_section("exporter", optional=True): minimal_addon_schema(),
+            addon_section("groups", optional=True): minimal_addon_schema(),
+            addon_section("meta_modeling", optional=True): minimal_addon_schema(),
+            addon_section("products", optional=True): minimal_addon_schema(),
+            addon_section("publications", optional=True): minimal_addon_schema(),
+            addon_section("remote_debug", optional=True): minimal_addon_schema(),
+            addon_section("security", optional=True): minimal_addon_schema(),
+            addon_section("statics", optional=True): minimal_addon_schema(),
+            addon_section("studies_access", optional=True): minimal_addon_schema(),
+            addon_section("studies_dispatcher", optional=True): minimal_addon_schema(),
+            addon_section("tags", optional=True): minimal_addon_schema(),
+            addon_section("users", optional=True): minimal_addon_schema(),
+            addon_section("version_control", optional=True): minimal_addon_schema(),
+        }
+    )
+
+    section_names = [k.name for k in schema.keys]
+
+    # fmt: off
+    assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}"  # nosec
+    # fmt: on
+
+    return schema
+
+
+def load_default_config(environs=None) -> Dict:
+    filepath: Path = resources.get_path(f"config/{CLI_DEFAULT_CONFIGFILE}")
+    return read_and_validate(filepath, trafaret=app_schema, vars=environs)
+
+
+app_schema = create_schema()

services/web/server/src/simcore_service_webserver/application_config.py

@@ -11,123 +11,15 @@
 
 """
 import logging
-from pathlib import Path
 from typing import Dict
 
-import trafaret as T
 from aiohttp.web import Application
 from models_library.basic_types import LogLevel, PortInt
 from pydantic import BaseSettings, Field
 from servicelib.aiohttp.application_keys import APP_CONFIG_KEY
-from trafaret_config.simple import read_and_validate
-
-from . import (
-    catalog_config,
-    db_config,
-    email_config,
-    rest_config,
-    session_config,
-    storage_config,
-    tracing,
-)
-from .activity import config as activity_config
-from .director import config as director_config
-from .login import config as login_config
-from .projects import config as projects_config
-from .resource_manager import config as resource_manager_config
-from .resources import resources
-from .socketio import config as socketio_config
 
 log = logging.getLogger(__name__)
 
-CLI_DEFAULT_CONFIGFILE = "server-defaults.yaml"
-assert resources.exists("config/" + CLI_DEFAULT_CONFIGFILE)  # nosec
-
-
-def addon_section(name: str, optional: bool = False) -> T.Key:
-    if optional:
-        return T.Key(name, default=dict(enabled=True), optional=optional)
-    return T.Key(name)
-
-
-def minimal_addon_schema() -> T.Dict:
-    return T.Dict({T.Key("enabled", default=True, optional=True): T.Bool()})
-
-
-def create_schema() -> T.Dict:
-    """
-    Build schema for the configuration's file
-    by aggregating all the subsystem configurations
-    """
-    # pylint: disable=protected-access
-    schema = T.Dict(
-        {
-            "version": T.String(),
-            "main": T.Dict(
-                {
-                    "host": T.IP,
-                    "port": T.ToInt(),
-                    "log_level": T.Enum(*logging._nameToLevel.keys()),
-                    "testing": T.Bool(),
-                    T.Key("studies_access_enabled", default=False): T.Or(
-                        T.Bool(), T.ToInt
-                    ),
-                }
-            ),
-            addon_section(tracing.CONFIG_SECTION_NAME, optional=True): tracing.schema,
-            db_config.CONFIG_SECTION_NAME: db_config.schema,
-            director_config.CONFIG_SECTION_NAME: director_config.schema,
-            rest_config.CONFIG_SECTION_NAME: rest_config.schema,
-            projects_config.CONFIG_SECTION_NAME: projects_config.schema,
-            email_config.CONFIG_SECTION_NAME: email_config.schema,
-            storage_config.CONFIG_SECTION_NAME: storage_config.schema,
-            addon_section(
-                login_config.CONFIG_SECTION_NAME, optional=True
-            ): login_config.schema,
-            addon_section(
-                socketio_config.CONFIG_SECTION_NAME, optional=True
-            ): socketio_config.schema,
-            session_config.CONFIG_SECTION_NAME: session_config.schema,
-            activity_config.CONFIG_SECTION_NAME: activity_config.schema,
-            resource_manager_config.CONFIG_SECTION_NAME: resource_manager_config.schema,
-            # BELOW HERE minimal sections until more options are needed
-            addon_section("catalog", optional=True): catalog_config.schema,
-            addon_section("clusters", optional=True): minimal_addon_schema(),
-            addon_section("computation", optional=True): minimal_addon_schema(),
-            addon_section("diagnostics", optional=True): minimal_addon_schema(),
-            addon_section("director-v2", optional=True): minimal_addon_schema(),
-            addon_section("exporter", optional=True): minimal_addon_schema(),
-            addon_section("groups", optional=True): minimal_addon_schema(),
-            addon_section("meta_modeling", optional=True): minimal_addon_schema(),
-            addon_section("products", optional=True): minimal_addon_schema(),
-            addon_section("publications", optional=True): minimal_addon_schema(),
-            addon_section("remote_debug", optional=True): minimal_addon_schema(),
-            addon_section("security", optional=True): minimal_addon_schema(),
-            addon_section("statics", optional=True): minimal_addon_schema(),
-            addon_section("studies_access", optional=True): minimal_addon_schema(),
-            addon_section("studies_dispatcher", optional=True): minimal_addon_schema(),
-            addon_section("tags", optional=True): minimal_addon_schema(),
-            addon_section("users", optional=True): minimal_addon_schema(),
-            addon_section("version_control", optional=True): minimal_addon_schema(),
-        }
-    )
-
-    section_names = [k.name for k in schema.keys]
-
-    # fmt: off
-    assert len(section_names) == len(set(section_names)), f"Found repeated section names in {section_names}"  # nosec
-    # fmt: on
-
-    return schema
-
-
-def load_default_config(environs=None) -> Dict:
-    filepath: Path = resources.get_path(f"config/{CLI_DEFAULT_CONFIGFILE}")
-    return read_and_validate(filepath, trafaret=app_schema, vars=environs)
-
-
-app_schema = create_schema()
-
 
 class MainSettings(BaseSettings):
     host: str = "0.0.0.0"  # nosec

services/web/server/src/simcore_service_webserver/catalog.py

@@ -17,11 +17,8 @@
     is_service_responsive,
     to_backend_service,
 )
-from .catalog_config import (
-    KCATALOG_ORIGIN,
-    KCATALOG_VERSION_PREFIX,
-    assert_valid_config,
-)
+from .catalog_config import KCATALOG_ORIGIN, KCATALOG_VERSION_PREFIX
+from .catalog_settings import assert_valid_config
 from .login.decorators import RQT_USERID_KEY, login_required
 from .security_decorators import permission_required