♻️ update settings api server (#2563)

- ♻️ refactors api-server to use settings-library (based on pattern presented in http://github.com/pcrespov/sample-app-settings)
- ♻️ formatted meta constants to uppercase in both api-server and director-v2
- ✨adds CLI command to resolve and print envfiles
- 🐛 Added a retry policy to retrieve index from static web server since i noticed that unhandled exception shut down the web-server in one of the tests


* Add settings-library to requirements

* directorv2: formatted all constants in meta in upper-case

* api-server: formatted all constants in meta in upper-case

* fixes reqs

* api-server changed to new settings

* add cli

* missing setting

* FIX: disable scicrunch unreliable test

* FIX: Retries connection web->static services

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -13,7 +13,7 @@
 from copy import deepcopy
 from pathlib import Path
 from pprint import pformat
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Union
 
 import pytest
 import yaml
@@ -27,7 +27,7 @@
 
 
 @pytest.fixture(scope="session")
-def testing_environ_vars(env_devel_file: Path) -> Dict[str, str]:
+def testing_environ_vars(env_devel_file: Path) -> Dict[str, Union[str, None]]:
     """
     Loads and extends .env-devel returning
     all environment variables key=value

packages/settings-library/src/settings_library/postgres.py

@@ -53,6 +53,12 @@ def dsn(self) -> str:
             port=f"{self.POSTGRES_PORT}",
             path=f"/{self.POSTGRES_DB}",
         )
+        return dsn
+
+    @cached_property
+    def dsn_with_query(self) -> str:
+        """Some clients do not support queries in the dsn"""
+        dsn = self.dsn
         if self.POSTGRES_CLIENT_NAME:
             dsn += "?" + urllib.parse.urlencode(
                 {"application_name": self.POSTGRES_CLIENT_NAME}

packages/settings-library/tests/test_postgres.py

@@ -43,6 +43,6 @@ def test_dsn_with_query(mock_environment: Dict, monkeypatch):
 
     assert settings_with_app.POSTGRES_CLIENT_NAME
     assert (
-        settings_with_app.dsn
+        settings_with_app.dsn_with_query
         == "postgresql://foo:secret@localhost:5432/foodb?application_name=Some+%2643+funky+name"
     )

services/api-server/.env-devel

@@ -1,6 +1,17 @@
 #
 # Environment variables used to configure this service
 #
+#
+# - Keep it alfphabetical order and grouped by prefix [see vscode cmd: Sort Lines Ascending]
+# - To expose:
+#     set -o allexport
+#     source .env
+#     set +o allexport
+#   or
+#     export $(grep -v '^#' .env | xargs)
+#     unset  $(grep -v '^#' .env | sed -E 's/(.*)=.*/\1/' | xargs)
+#
+
 API_SERVER_DEV_FEATURES_ENABLED=1
 DEBUG=0
 
@@ -20,14 +31,12 @@ SC_BOOT_MODE=debug-ptvsd
 
 
 # webserver
-WEBSERVER_ENABLED=1
 WEBSERVER_HOST=webserver
 # Take from general .env-devel
-WEBSERVER_SESSION_SECRET_KEY=REPLACE ME with a key of at least length 32.
+WEBSERVER_SESSION_SECRET_KEY="REPLACE ME with a key of at least length 32."
 
 
 # catalog
-CATALOG_ENABLED=1
 CATALOG_HOST=catalog
 
 # storage

services/api-server/requirements/_base.in

@@ -8,6 +8,7 @@
 # intra-repo required dependencies
 --requirement ../../../packages/models-library/requirements/_base.in
 --requirement ../../../packages/postgres-database/requirements/_base.in
+--requirement ../../../packages/settings-library/requirements/_base.in
 --requirement ../../../packages/simcore-sdk/requirements/_base.in
 # service-library[fastapi]
 --requirement ../../../packages/service-library/requirements/_base.in

services/api-server/requirements/_base.txt

@@ -19,6 +19,7 @@ aiohttp==3.7.4.post0
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
@@ -62,13 +63,15 @@ chardet==4.0.0
 click==7.1.2
     # via
     #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_migration.in
+    #   typer
     #   uvicorn
 cryptography==3.4.7
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
@@ -130,6 +133,7 @@ importlib-resources==5.1.3 ; python_version < "3.9"
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
@@ -147,6 +151,7 @@ jinja2==2.11.3
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
@@ -195,13 +200,15 @@ pydantic==1.8.2
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   -r requirements/../../../packages/settings-library/requirements/_base.in
     #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/../../../packages/simcore-sdk/requirements/_base.in
@@ -229,6 +236,7 @@ pyyaml==5.4.1
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
@@ -266,6 +274,7 @@ sqlalchemy==1.3.24
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/_base.txt
@@ -300,6 +309,8 @@ trafaret==2.1.0
     # via trafaret-config
 trafaret-config==2.0.2
     # via -r requirements/../../../packages/simcore-sdk/requirements/_base.in
+typer==0.4.0
+    # via -r requirements/../../../packages/settings-library/requirements/_base.in
 typing-extensions==3.10.0.2
     # via
     #   aiohttp
@@ -316,6 +327,7 @@ urllib3==1.26.5
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
@@ -329,6 +341,7 @@ uvicorn==0.13.4
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/simcore-sdk/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt

services/api-server/requirements/ci.txt

@@ -16,6 +16,7 @@
 ../../packages/pytest-simcore/
 ../../packages/simcore-sdk
 ../../packages/service-library[fastapi]
+../../packages/settings-library/
 ../../services/storage/client-sdk/python/
 
 # installs current package

services/api-server/requirements/dev.txt

@@ -15,9 +15,9 @@
 --editable ../../packages/models-library
 --editable ../../packages/postgres-database
 --editable ../../packages/pytest-simcore/
---editable ../../packages/simcore-sdk
 --editable ../../packages/service-library[fastapi]
+--editable ../../packages/settings-library/
+--editable ../../packages/simcore-sdk
 --editable ../../services/storage/client-sdk/python/
-
 # installs current package
 --editable .

services/api-server/requirements/prod.txt

@@ -12,9 +12,9 @@
 # installs this repo's packages
 ../../packages/models-library
 ../../packages/postgres-database/
-../../packages/simcore-sdk
 ../../packages/service-library[fastapi]
+../../packages/settings-library/
+../../packages/simcore-sdk
 ../../services/storage/client-sdk/python/
-
 # installs current package
 .

services/api-server/setup.py

@@ -33,6 +33,7 @@ def read_reqs(reqs_path: Path) -> Set[str]:
         "simcore-postgres-database",
         "simcore-sdk",
         "simcore-service-library[fastapi]",
+        "simcore-settings-library",
     }
 )
 
@@ -63,4 +64,9 @@ def read_reqs(reqs_path: Path) -> Set[str]:
         test_suite="tests",
         tests_require=TEST_REQUIREMENTS,
         extras_require={"test": TEST_REQUIREMENTS},
+        entry_points={
+            "console_scripts": [
+                "simcore-service-api-server = simcore_service_api_server.cli:main",
+            ],
+        },
     )

services/api-server/src/simcore_service_api_server/_meta.py

@@ -5,26 +5,26 @@
 
 import pkg_resources
 
-current_distribution = pkg_resources.get_distribution("simcore_service_api_server")
+_current_distribution = pkg_resources.get_distribution("simcore_service_api_server")
 
-project_name: str = current_distribution.project_name
+PROJECT_NAME: str = _current_distribution.project_name
 
-api_version: str = current_distribution.version
-major, minor, patch = current_distribution.version.split(".")
-api_vtag: str = f"v{major}"
+API_VERSION: str = _current_distribution.version
+MAJOR, MINOR, PATCH = _current_distribution.version.split(".")
+API_VTAG: str = f"v{MAJOR}"
 
-__version__ = current_distribution.version
+__version__ = _current_distribution.version
 
 
 def get_summary() -> str:
     with suppress(Exception):
         try:
-            metadata = current_distribution.get_metadata_lines("METADATA")
+            metadata = _current_distribution.get_metadata_lines("METADATA")
         except FileNotFoundError:
-            metadata = current_distribution.get_metadata_lines("PKG-INFO")
+            metadata = _current_distribution.get_metadata_lines("PKG-INFO")
 
         return next(x.split(":") for x in metadata if x.startswith("Summary:"))[-1]
     return ""
 
 
-summary: str = get_summary()
+SUMMARY: str = get_summary()

services/api-server/src/simcore_service_api_server/api/dependencies/webserver.py

@@ -6,7 +6,7 @@
 from fastapi import Depends, FastAPI, HTTPException, status
 from fastapi.requests import Request
 
-from ...core.settings import AppSettings, WebServerSettings
+from ...core.settings import WebServerSettings
 from ...modules.webserver import AuthSession
 from .authentication import get_active_user_email
 
@@ -18,8 +18,12 @@ def _get_app(request: Request) -> FastAPI:
 
 
 def _get_settings(request: Request) -> WebServerSettings:
-    app_settings: AppSettings = request.app.state.settings
-    return app_settings.webserver
+    s = request.app.state.settings.API_SERVER_WEBSERVER
+    if not s:
+        raise HTTPException(
+            status.HTTP_503_SERVICE_UNAVAILABLE, detail="webserver disabled"
+        )
+    return s
 
 
 def _get_encrypt(request: Request) -> Optional[Fernet]:
@@ -38,7 +42,7 @@ def get_session_cookie(
         raise HTTPException(status.HTTP_503_SERVICE_UNAVAILABLE, detail=UNAVAILBLE_MSG)
 
     # builds session cookie
-    cookie_name = settings.session_name
+    cookie_name = settings.WEBSERVER_SESSION_NAME
     cookie_data = json.dumps(
         {
             "created": int(time.time()),  # now
@@ -57,7 +61,7 @@ def get_webserver_session(
     session_cookies: Dict = Depends(get_session_cookie),
 ) -> AuthSession:
     """
-        Lifetime of AuthSession wrapper is one request because it needs different session cookies
-        Lifetime of embedded client is attached to the app lifetime
+    Lifetime of AuthSession wrapper is one request because it needs different session cookies
+    Lifetime of embedded client is attached to the app lifetime
     """
     return AuthSession.create(app, session_cookies)

services/api-server/src/simcore_service_api_server/api/root.py

@@ -12,7 +12,7 @@ def create_router(settings: AppSettings):
     router.include_router(meta.router, tags=["meta"], prefix="/meta")
     router.include_router(users.router, tags=["users"], prefix="/me")
 
-    if settings.dev_features_enabled:
+    if settings.API_SERVER_DEV_FEATURES_ENABLED:
         router.include_router(files.router, tags=["files"], prefix="/files")
         router.include_router(solvers.router, tags=["solvers"], prefix="/solvers")
         router.include_router(solvers_jobs.router, tags=["solvers"], prefix="/solvers")

services/api-server/src/simcore_service_api_server/api/routes/files.py

@@ -16,7 +16,7 @@
 from pydantic import ValidationError
 from starlette.responses import RedirectResponse
 
-from ..._meta import api_vtag
+from ..._meta import API_VTAG
 from ...models.schemas.files import File
 from ...modules.storage import StorageApi, StorageFileMetaData, to_file_api_model
 from ..dependencies.authentication import get_current_user_id
@@ -218,7 +218,7 @@ async def files_upload_multiple_view():
         content=dedent(
             f"""
         <body>
-        <form action="/{api_vtag}/files:upload" enctype="multipart/form-data" method="post">
+        <form action="/{API_VTAG}/files:upload" enctype="multipart/form-data" method="post">
         <input name="files" type="file" multiple>
         <input type="submit">
         </form>

services/api-server/src/simcore_service_api_server/api/routes/health.py

@@ -6,7 +6,7 @@
 from fastapi.responses import PlainTextResponse
 from models_library.app_diagnostics import AppStatusCheck
 
-from ..._meta import api_version, project_name
+from ..._meta import API_VERSION, PROJECT_NAME
 from ...modules.catalog import CatalogApi
 from ...modules.director_v2 import DirectorV2Api
 from ...modules.storage import StorageApi
@@ -34,8 +34,8 @@ async def get_service_state(
     heaths: Tuple[bool] = await asyncio.gather(*[api.is_responsive() for api in apis])
 
     current_status = AppStatusCheck(
-        app_name=project_name,
-        version=api_version,
+        app_name=PROJECT_NAME,
+        version=API_VERSION,
         services={
             api.service_name: {
                 "healthy": is_healty,

services/api-server/src/simcore_service_api_server/api/routes/meta.py

@@ -2,7 +2,7 @@
 
 from fastapi import APIRouter, Depends
 
-from ..._meta import __version__, api_version, api_vtag
+from ..._meta import API_VERSION, API_VTAG, __version__
 from ...models.schemas.meta import Meta
 from ..dependencies.application import get_reverse_url_mapper
 
@@ -15,8 +15,8 @@ async def get_service_metadata(
 ):
     return Meta(
         name=__name__.split(".")[0],
-        version=api_version,
-        released={api_vtag: api_version},
+        version=API_VERSION,
+        released={API_VTAG: API_VERSION},
         docs_url=url_for("redoc_html"),
         docs_dev_url=url_for("swagger_ui_html"),
     )

services/api-server/src/simcore_service_api_server/cli.py

@@ -0,0 +1,23 @@
+import logging
+
+import typer
+from settings_library.cli_utils import create_settings_command
+
+from ._meta import PROJECT_NAME
+from .core.settings import AppSettings
+
+log = logging.getLogger(__name__)
+main = typer.Typer(name=PROJECT_NAME)
+
+
+main.command()(create_settings_command(settings_cls=AppSettings, logger=log))
+
+
+@main.command()
+def run():
+    """Runs application"""
+    typer.secho("Sorry, this entrypoint is intentionally disabled." "Use instead")
+    typer.secho(
+        "$ uvicorn simcore_service_api_server.main:the_app",
+        fg=typer.colors.BLUE,
+    )