✨ Is2805/can disable exporter with envs (⚠️ devops) (#2814)

services/web/server/src/simcore_service_webserver/application_settings.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
 
 from aiohttp import web
 from models_library.basic_types import (
@@ -152,8 +152,27 @@ class Config(BaseCustomSettings.Config):
 
     # HELPERS  --------------------------------------------------------
 
-    def is_enabled(self, plugin_name: str):
-        return getattr(self, f"WEBSERVER_{plugin_name.upper()}", None) is not None
+    def is_enabled(self, field_name: str) -> bool:
+        return getattr(self, field_name, None) is not None
+
+    def is_plugin(self, field_name: str) -> bool:
+        if field := self.__fields__.get(field_name):
+            if "auto_default_from_env" in field.field_info.extra and field.allow_none:
+                return True
+        return False
+
+    def _get_disabled_public_plugins(self) -> List[str]:
+        plugins_disabled = []
+        # NOTE: this list is limited for security reasons. An unbounded list
+        # might reveal critical info on the settings of a deploy to the client.
+        PUBLIC_PLUGIN_CANDIDATES = [
+            "WEBSERVER_EXPORTER",
+            "WEBSERVER_SCICRUNCH",
+        ]
+        for field_name in PUBLIC_PLUGIN_CANDIDATES:
+            if self.is_plugin(field_name) and not self.is_enabled(field_name):
+                plugins_disabled.append(field_name)
+        return plugins_disabled
 
     def public_dict(self) -> Dict[str, Any]:
         """Data publicaly available"""
@@ -182,6 +201,8 @@ def to_client_statics(self) -> Dict[str, Any]:
             exclude_none=True,
             by_alias=True,
         )
+        data["plugins_disabled"] = self._get_disabled_public_plugins()
+
         # Alias in addition MUST be camelcase here
         return {snake_to_camel(k): v for k, v in data.items()}
 
@@ -317,9 +338,9 @@ def convert_to_app_config(app_settings: ApplicationSettings) -> Dict[str, Any]:
             ),
         },
         "clusters": {"enabled": True},
-        "computation": {"enabled": app_settings.is_enabled("COMPUTATION")},
-        "diagnostics": {"enabled": app_settings.is_enabled("DIAGNOSTICS")},
-        "director-v2": {"enabled": app_settings.is_enabled("DIRECTOR_V2")},
+        "computation": {"enabled": app_settings.is_enabled("WEBSERVER_COMPUTATION")},
+        "diagnostics": {"enabled": app_settings.is_enabled("WEBSERVER_DIAGNOSTICS")},
+        "director-v2": {"enabled": app_settings.is_enabled("WEBSERVER_DIRECTOR_V2")},
         "exporter": {"enabled": app_settings.WEBSERVER_EXPORTER is not None},
         "groups": {"enabled": True},
         "meta_modeling": {"enabled": True},

services/web/server/src/simcore_service_webserver/exporter/file_downloader.py

@@ -28,6 +28,10 @@ async def append_file(self, link: str, download_path: Path) -> None:
     async def download_files(self, app: Application) -> None:
         """starts the download and waits for all files to finish"""
         exporter_settings = get_settings(app)
+        assert (  # nosec
+            exporter_settings is not None
+        ), "this call was not expected with a disabled plugin"  # nosec
+
         results = await self.downloader.run_download(
             timeouts={
                 "total": exporter_settings.EXPORTER_DOWNLOADER_MAX_TIMEOUT_SECONDS,

services/web/server/src/simcore_service_webserver/exporter/module_setup.py

@@ -1,14 +1,19 @@
 import logging
 
 from aiohttp import web
-from servicelib.aiohttp.application_setup import ModuleCategory, app_module_setup
+from servicelib.aiohttp.application_setup import (
+    ModuleCategory,
+    SkipModuleSetup,
+    app_module_setup,
+)
 from servicelib.aiohttp.rest_routing import (
     iter_path_operations,
     map_handlers_with_operations,
 )
 
 from .._constants import APP_OPENAPI_SPECS_KEY
 from .request_handlers import rest_handler_functions
+from .settings import get_settings
 
 logger = logging.getLogger(__name__)
 
@@ -20,6 +25,16 @@
 )
 def setup_exporter(app: web.Application) -> bool:
 
+    # TODO: Implements temporary plugin disabling mechanims until new settings are fully integrated in servicelib.aiohttp.app_module_setup
+    try:
+        if get_settings(app) is None:
+            raise SkipModuleSetup(
+                reason="{__name__} plugin was explictly disabled in the app settings"
+            )
+    except KeyError as err:
+        # This will happen if app[APP_SETTINGS_KEY] raises
+        raise SkipModuleSetup(reason="{__name__} plugin settings undefined") from err
+
     # Rest-API routes: maps handlers with routes tags with "viewer" based on OAS operation_id
     specs = app[APP_OPENAPI_SPECS_KEY]
     rest_routes = map_handlers_with_operations(

services/web/server/src/simcore_service_webserver/exporter/request_handlers.py

@@ -88,6 +88,10 @@ async def import_project(request: web.Request):
     # bumping this requests's max size
     # pylint: disable=protected-access
     exporter_settings = get_settings(request.app)
+    assert (  # nosec
+        exporter_settings is not None
+    ), "this call was not expected with a disabled plugin"  # nosec
+
     request._client_max_size = exporter_settings.EXPORTER_MAX_UPLOAD_FILE_SIZE * ONE_GB
 
     post_contents = await request.post()

services/web/server/src/simcore_service_webserver/exporter/settings.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from aiohttp.web import Application
 from pydantic import Field, PositiveInt
 from servicelib.aiohttp.application_keys import APP_SETTINGS_KEY
@@ -23,7 +25,6 @@ class ExporterSettings(BaseCustomSettings):
     )
 
 
-def get_settings(app: Application) -> ExporterSettings:
+def get_settings(app: Application) -> Optional[ExporterSettings]:
     settings = app[APP_SETTINGS_KEY].WEBSERVER_EXPORTER
-    assert settings  # nosec
     return settings

services/web/server/tests/integration/01/test_exporter.py

@@ -67,6 +67,9 @@
 from simcore_service_webserver.db_models import projects
 from simcore_service_webserver.exporter.async_hashing import Algorithm, checksum
 from simcore_service_webserver.exporter.file_downloader import ParallelDownloader
+from simcore_service_webserver.exporter.settings import (
+    get_settings as get_exporter_settings,
+)
 from simcore_service_webserver.scicrunch.submodule_setup import (
     setup_scicrunch_submodule,
 )
@@ -141,19 +144,24 @@ def client(
     mock_orphaned_services: mock.Mock,
     monkeypatch_setenv_from_app_config: Callable,
 ):
+    # test config & env vars ----------------------
     cfg = deepcopy(app_config)
-
     assert cfg["rest"]["version"] == API_VERSION
     assert cfg["rest"]["enabled"]
+
     cfg["projects"]["enabled"] = True
     cfg["director"]["enabled"] = True
+    cfg["exporter"]["enabled"] = True
 
-    # fake config
     monkeypatch_setenv_from_app_config(cfg)
+
+    # app setup ----------------------------------
     app = create_safe_application(cfg)
 
     # activates only security+restAPI sub-modules
     setup_settings(app)
+    assert get_exporter_settings(app) is not None, "Should capture defaults"
+
     setup_db(app)
     setup_session(app)
     setup_security(app)
@@ -164,7 +172,7 @@ def client(
     setup_projects(app)
     setup_director(app)
     setup_director_v2(app)
-    setup_exporter(app)
+    setup_exporter(app)  # <---- under test
     setup_storage(app)
     setup_products(app)
     setup_catalog(app)

services/web/server/tests/unit/isolated/test_application_settings.py

@@ -208,7 +208,10 @@ def test_settings_constructs(app_settings: ApplicationSettings):
 
 
 def test_settings_to_client_statics(app_settings: ApplicationSettings):
+
     statics = app_settings.to_client_statics()
+    # can jsonify
+    print(json.dumps(statics, indent=1))
 
     # all key in camelcase
     assert all(
@@ -218,9 +221,22 @@ def test_settings_to_client_statics(app_settings: ApplicationSettings):
 
     # special alias
     assert statics["stackName"] == "master-simcore"
+    assert not statics["pluginsDisabled"]
 
-    # can jsonify
-    print(json.dumps(statics))
+
+def test_settings_to_client_statics_plugins(
+    mock_webserver_service_environment, monkeypatch
+):
+    disable_plugins = {"WEBSERVER_EXPORTER", "WEBSERVER_SCICRUNCH"}
+    for name in disable_plugins:
+        monkeypatch.setenv(name, "null")
+
+    settings = ApplicationSettings()
+    statics = settings.to_client_statics()
+
+    print(json.dumps(statics, indent=1))
+
+    assert set(statics["pluginsDisabled"]) == disable_plugins
 
 
 def test_avoid_sensitive_info_in_public(app_settings: ApplicationSettings):