♻️ gc as a service (preparation) (#2828)

.codeclimate.yml

@@ -56,7 +56,7 @@ plugins:
     channel: "eslint-6"
     config:
       extensions:
-      - .js
+        - .js
 
 exclude_patterns:
   - "config/"

packages/service-library/src/servicelib/aiohttp/application_setup.py

@@ -5,15 +5,15 @@
 from datetime import datetime
 from distutils.util import strtobool
 from enum import Enum
-from typing import Any, Callable, Dict, List, Optional, Protocol
+from typing import Any, Callable, Dict, List, Optional, Protocol, Tuple
 
 from aiohttp import web
 
-from .application_keys import APP_CONFIG_KEY
+from .application_keys import APP_CONFIG_KEY, APP_SETTINGS_KEY
 
 log = logging.getLogger(__name__)
 
-APP_SETUP_KEY = f"{__name__ }.setup"
+APP_SETUP_COMPLETED_KEY = f"{__name__ }.setup"
 
 
 class _SetupFunc(Protocol):
@@ -23,44 +23,101 @@ def __call__(self, app: web.Application, *args: Any, **kwds: Any) -> bool:
         ...
 
 
+class _ApplicationSettings(Protocol):
+    def is_enabled(self, field_name: str) -> bool:
+        ...
+
+
 class ModuleCategory(Enum):
     SYSTEM = 0
     ADDON = 1
 
 
+# ERRORS ------------------------------------------------------------------
+
+
 class SkipModuleSetup(Exception):
     def __init__(self, *, reason) -> None:
         self.reason = reason
         super().__init__(reason)
 
 
 class ApplicationSetupError(Exception):
-    pass
+    ...
 
 
 class DependencyError(ApplicationSetupError):
-    pass
+    ...
+
+
+# HELPERS ------------------------------------------------------------------
+
+
+def _is_addon_enabled_from_config(
+    cfg: Dict[str, Any], dotted_section: str, section
+) -> bool:
+    try:
+        parts: List[str] = dotted_section.split(".")
+        # navigates app_config (cfg) searching for section
+        searched_config = deepcopy(cfg)
+        for part in parts:
+            if section and part == "enabled":
+                # if section exists, no need to explicitly enable it
+                return strtobool(f"{searched_config.get(part, True)}")
+            searched_config = searched_config[part]
 
+    except KeyError as ee:
+        raise ApplicationSetupError(
+            f"Cannot find required option '{dotted_section}' in app config's section '{ee}'"
+        ) from ee
+    else:
+        assert isinstance(searched_config, bool)  # nosec
+        return searched_config
+
+
+def _get_app_settings_and_field_name(
+    app: web.Application,
+    arg_module_name: str,
+    arg_settings_name: Optional[str],
+    setup_func_name: str,
+    logger: logging.Logger,
+) -> Tuple[Optional[_ApplicationSettings], Optional[str]]:
+
+    app_settings: Optional[_ApplicationSettings] = app.get(APP_SETTINGS_KEY)
+    settings_field_name = arg_settings_name
+
+    if app_settings:
+
+        if not settings_field_name:
+            # FIXME: hard-coded WEBSERVER_ temporary
+            settings_field_name = f"WEBSERVER_{arg_module_name.split('.')[-1].upper()}"
+
+        logger.debug("Checking addon's %s ", f"{settings_field_name=}")
+
+        if not hasattr(app_settings, settings_field_name):
+            raise ValueError(
+                f"Invalid option {arg_settings_name=} in module's setup {setup_func_name}. "
+                f"It must be a field in {app_settings.__class__}"
+            )
 
-def _is_app_module_enabled(cfg: Dict, parts: List[str], section) -> bool:
-    # navigates app_config (cfg) searching for section
-    searched_config = deepcopy(cfg)
-    for part in parts:
-        if section and part == "enabled":
-            # if section exists, no need to explicitly enable it
-            return strtobool(f"{searched_config.get(part, True)}")
-        searched_config = searched_config[part]
-    assert isinstance(searched_config, bool)  # nosec
-    return searched_config
+    return app_settings, settings_field_name
+
+
+# PUBLIC API ------------------------------------------------------------------
+
+
+def is_setup_completed(module_name: str, app: web.Application) -> bool:
+    return module_name in app[APP_SETUP_COMPLETED_KEY]
 
 
 def app_module_setup(
     module_name: str,
     category: ModuleCategory,
     *,
     depends: Optional[List[str]] = None,
-    config_section: str = None,
-    config_enabled: str = None,
+    config_section: Optional[str] = None,
+    config_enabled: Optional[str] = None,
+    settings_name: Optional[str] = None,
     logger: logging.Logger = log,
 ) -> Callable:
     """Decorator that marks a function as 'a setup function' for a given module in an application
@@ -77,6 +134,7 @@ def app_module_setup(
     :param depends: list of module_names that must be called first, defaults to None
     :param config_section: explicit configuration section, defaults to None (i.e. the name of the module, or last entry of the name if dotted)
     :param config_enabled: option in config to enable, defaults to None which is '$(module-section).enabled' (config_section and config_enabled are mutually exclusive)
+    :param settings_name: field name in the app's settings that corresponds to this module. Defaults to the name of the module with app prefix.
     :raises DependencyError
     :raises ApplicationSetupError
     :return: True if setup was completed or False if setup was skipped
@@ -111,7 +169,7 @@ def _decorate(setup_func: _SetupFunc):
             logger.warning("Rename '%s' to contain 'setup'", setup_func.__name__)
 
         # metadata info
-        def setup_metadata() -> Dict:
+        def setup_metadata() -> Dict[str, Any]:
             return {
                 "module_name": module_name,
                 "dependencies": depends,
@@ -132,56 +190,74 @@ def _wrapper(app: web.Application, *args, **kargs) -> bool:
                 f"{depends}",
             )
 
-            if APP_SETUP_KEY not in app:
-                app[APP_SETUP_KEY] = []
+            if APP_SETUP_COMPLETED_KEY not in app:
+                app[APP_SETUP_COMPLETED_KEY] = []
 
             if category == ModuleCategory.ADDON:
                 # NOTE: ONLY addons can be enabled/disabled
-                # TODO: sometimes section is optional, check in config schema
-                cfg = app[APP_CONFIG_KEY]
-
-                try:
-                    is_enabled = _is_app_module_enabled(
-                        cfg, config_enabled.split("."), section
-                    )
-                except KeyError as ee:
-                    raise ApplicationSetupError(
-                        f"Cannot find required option '{config_enabled}' in app config's section '{ee}'"
-                    ) from ee
 
+                # TODO: cfg will be fully replaced by app_settings section below
+                cfg = app[APP_CONFIG_KEY]
+                is_enabled = _is_addon_enabled_from_config(cfg, config_enabled, section)
                 if not is_enabled:
                     logger.info(
                         "Skipping '%s' setup. Explicitly disabled in config",
                         module_name,
                     )
                     return False
 
+                # NOTE: if not disabled by config, it can be disabled by settings (tmp while legacy maintained)
+                app_settings, module_settings_name = _get_app_settings_and_field_name(
+                    app,
+                    module_name,
+                    settings_name,
+                    setup_func.__name__,
+                    logger,
+                )
+
+                if (
+                    app_settings
+                    and module_settings_name
+                    and not app_settings.is_enabled(module_settings_name)
+                ):
+                    logger.info(
+                        "Skipping setup %s. %s disabled in settings",
+                        f"{module_name=}",
+                        f"{module_settings_name=}",
+                    )
+                    return False
+
             if depends:
+                # TODO: no need to enforce. Use to deduce order instead.
                 uninitialized = [
-                    dep for dep in depends if dep not in app[APP_SETUP_KEY]
+                    dep for dep in depends if not is_setup_completed(dep, app)
                 ]
                 if uninitialized:
-                    msg = f"Cannot setup app module '{module_name}' because the following dependencies are still uninitialized: {uninitialized}"
-                    log.error(msg)
-                    raise DependencyError(msg)
-
-            if module_name in app[APP_SETUP_KEY]:
-                msg = f"'{module_name}' was already initialized in {app}. Setup can only be executed once per app."
-                logger.error(msg)
-                raise ApplicationSetupError(msg)
+                    raise DependencyError(
+                        f"Cannot setup app module '{module_name}' because the "
+                        f"following dependencies are still uninitialized: {uninitialized}"
+                    )
 
             # execution of setup
             try:
+                if is_setup_completed(module_name, app):
+                    raise SkipModuleSetup(
+                        reason=f"'{module_name}' was already initialized in {app}."
+                        " Setup can only be executed once per app."
+                    )
+
                 completed = setup_func(app, *args, **kargs)
 
                 # post-setup
                 if completed is None:
                     completed = True
 
-                if completed:
-                    app[APP_SETUP_KEY].append(module_name)
+                if completed:  # registers completed setup
+                    app[APP_SETUP_COMPLETED_KEY].append(module_name)
                 else:
-                    raise SkipModuleSetup(reason="Undefined")
+                    raise SkipModuleSetup(
+                        reason="Undefined (setup function returned false)"
+                    )
 
             except SkipModuleSetup as exc:
                 logger.warning("Skipping '%s' setup: %s", module_name, exc.reason)
@@ -197,17 +273,18 @@ def _wrapper(app: web.Application, *args, **kargs) -> bool:
             return completed
 
         _wrapper.metadata = setup_metadata
-        _wrapper.MARK = "setup"
+        _wrapper.mark_as_simcore_servicelib_setup_func = True
 
         return _wrapper
 
     return _decorate
 
 
-def is_setup_function(fun):
+def is_setup_function(fun: Callable) -> bool:
+    # TODO: use _SetupFunc protocol to check in runtime
     return (
         inspect.isfunction(fun)
-        and getattr(fun, "MARK", None) == "setup"
+        and hasattr(fun, "mark_as_simcore_servicelib_setup_func")
         and any(
             param.annotation == web.Application
             for _, param in inspect.signature(fun).parameters.items()

packages/service-library/tests/aiohttp/test_application_setup.py

@@ -9,11 +9,11 @@
 from aiohttp import web
 from servicelib.aiohttp.application_keys import APP_CONFIG_KEY
 from servicelib.aiohttp.application_setup import (
-    APP_SETUP_KEY,
     DependencyError,
     ModuleCategory,
     SkipModuleSetup,
     app_module_setup,
+    is_setup_completed,
 )
 
 log = Mock()
@@ -91,7 +91,7 @@ def test_marked_setup(app_config, app):
     assert setup_foo(app, 1)
 
     assert setup_foo.metadata()["module_name"] == "package.foo"
-    assert setup_foo.metadata()["module_name"] in app[APP_SETUP_KEY]
+    assert is_setup_completed(setup_foo.metadata()["module_name"], app)
 
     app_config["foo"]["enabled"] = False
     assert not setup_foo(app, 2)

services/docker-compose.devel.yml

@@ -60,16 +60,19 @@ services:
 
   webserver:
     volumes:
+      &webserver-volumes
       - ./web/server:/devel/services/web/server
       - ./web/client/source-output:/devel/services/web/client
       - ../packages:/devel/packages
     environment:
+      &webserver-environment
       - SC_BOOT_MODE=debug-ptvsd
       - WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS=15
       - WEBSERVER_LOGLEVEL=${LOG_LEVEL:-DEBUG}
 
   dask-sidecar:
-    volumes: &dev-dask-sidecar-volumes
+    volumes:
+      &dev-dask-sidecar-volumes
       - ./dask-sidecar:/devel/services/dask-sidecar
       - ../packages:/devel/packages
       - ${ETC_HOSTNAME:-/etc/hostname}:/home/scu/hostname:ro

services/docker-compose.yml

@@ -358,8 +358,7 @@ services:
       - default
 
   postgres:
-    image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27\
-      c02d8adb8feb20f"
+    image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f"
     init: true
     hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
     environment:
@@ -414,8 +413,7 @@ services:
       ]
 
   redis:
-    image: "redis:5.0.9-alpine@sha256:b011c1ca7fa97ed92d6c5995e5dd752dc37fe157c1b60\
-      ce96a6e35701851dabc"
+    image: "redis:5.0.9-alpine@sha256:b011c1ca7fa97ed92d6c5995e5dd752dc37fe157c1b60ce96a6e35701851dabc"
     init: true
     hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
     networks:

services/web/server/src/simcore_service_webserver/application.py

@@ -20,12 +20,14 @@
 from .director_v2 import setup_director_v2
 from .email import setup_email
 from .exporter.module_setup import setup_exporter
+from .garbage_collector import setup_garbage_collector
 from .groups import setup_groups
 from .login.module_setup import setup_login
 from .meta_modeling import setup_meta_modeling
 from .products import setup_products
 from .projects.module_setup import setup_projects
 from .publications import setup_publications
+from .redis import setup_redis
 from .remote_debug import setup_remote_debugging
 from .resource_manager.module_setup import setup_resource_manager
 from .rest import setup_rest
@@ -75,9 +77,17 @@ def create_application(config: Dict[str, Any]) -> web.Application:
     setup_computation(app)
     setup_socketio(app)
     setup_login(app)
+
+    # interaction with other backend services
     setup_director(app)
     setup_director_v2(app)
     setup_storage(app)
+    setup_catalog(app)
+    setup_redis(app)
+
+    # resource management
+    setup_resource_manager(app)
+    setup_garbage_collector(app)
 
     # users
     setup_users(app)
@@ -91,9 +101,7 @@ def create_application(config: Dict[str, Any]) -> web.Application:
 
     # TODO: classify
     setup_activity(app)
-    setup_resource_manager(app)
     setup_tags(app)
-    setup_catalog(app)
     setup_publications(app)
     setup_products(app)
     setup_studies_access(app)