♻️ Maintenance/improve public api ci job

ci/github/system-testing/public-api.bash

@@ -28,7 +28,12 @@ install() {
 }
 
 test() {
-  pytest --color=yes --cov-report=term-missing -v --keep-docker-up tests/public-api --log-level=DEBUG
+  pytest \
+    --color=yes \
+    --cov-report=term-missing \
+    --keep-docker-up \
+    -v \
+    tests/public-api
 }
 
 clean_up() {

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -13,7 +13,7 @@
 from copy import deepcopy
 from pathlib import Path
 from pprint import pformat
-from typing import Any, Dict, List, Union
+from typing import Any, Dict, Iterator, List
 
 import pytest
 import yaml
@@ -25,18 +25,19 @@
     FIXTURE_CONFIG_OPS_SERVICES_SELECTION,
 )
 from .helpers.utils_docker import get_ip, run_docker_compose_config, save_docker_infos
+from .helpers.utils_environs import EnvVarsDict
 
 
 @pytest.fixture(scope="session")
-def testing_environ_vars(env_devel_file: Path) -> Dict[str, Union[str, None]]:
+def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict:
     """
     Loads and extends .env-devel returning
     all environment variables key=value
     """
     env_devel_unresolved = dotenv_values(env_devel_file, verbose=True, interpolate=True)
 
     # get from environ if applicable
-    env_devel = {
+    env_devel: EnvVarsDict = {
         key: os.environ.get(key, value) for key, value in env_devel_unresolved.items()
     }
 
@@ -77,7 +78,7 @@ def env_file_for_testing(
     testing_environ_vars: Dict[str, str],
     temp_folder: Path,
     osparc_simcore_root_dir: Path,
-) -> Path:
+) -> Iterator[Path]:
     """Dumps all the environment variables into an $(temp_folder)/.env.test file
 
     Pass path as argument in 'docker-compose --env-file ... '

packages/pytest-simcore/src/pytest_simcore/docker_swarm.py

@@ -21,6 +21,7 @@
 from tenacity.wait import wait_exponential, wait_fixed
 
 from .helpers.utils_docker import get_ip
+from .helpers.utils_environs import EnvVarsDict
 
 log = logging.getLogger(__name__)
 
@@ -140,13 +141,14 @@ def docker_stack(
     core_docker_compose_file: Path,
     ops_docker_compose_file: Path,
     keep_docker_up: bool,
-    testing_environ_vars: Dict,
+    testing_environ_vars: EnvVarsDict,
 ) -> Iterator[Dict]:
 
     # WARNING: keep prefix "pytest-" in stack names
     core_stack_name = testing_environ_vars["SWARM_STACK_NAME"]
     ops_stack_name = "pytest-ops"
 
+    assert core_stack_name
     assert core_stack_name.startswith("pytest-")
     stacks = [
         (

packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py

@@ -4,10 +4,12 @@
 import re
 from copy import deepcopy
 from pathlib import Path
-from typing import Dict, List
+from typing import Dict, List, Union
 
 import yaml
 
+EnvVarsDict = Dict[str, Union[str, None]]
+
 VARIABLE_SUBSTITUTION = re.compile(r"\$\{(\w+)(?:(:{0,1}[-?]{0,1})(.*))?\}$")
 
 
@@ -106,7 +108,7 @@ def eval_service_environ(
     image_environ: Dict = None,
     *,
     use_env_devel=True
-) -> Dict:
+) -> EnvVarsDict:
     """Deduces a service environment with it runs in a stack from confirmation
 
     :param docker_compose_path: path to stack configuration
@@ -133,7 +135,7 @@ def eval_service_environ(
     image_environ = image_environ or {}
 
     # Environ expected in a running service
-    service_environ = {}
+    service_environ: EnvVarsDict = {}
     service_environ.update(image_environ)
     service_environ.update(service["environment"])
     return service_environ

packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py

@@ -11,7 +11,7 @@
 
 
 @pytest.fixture(scope="function")
-async def dask_scheduler_service(simcore_services, monkeypatch) -> Dict[str, Any]:
+async def dask_scheduler_service(simcore_services_ready, monkeypatch) -> Dict[str, Any]:
     # the dask scheduler has a UI for the dashboard and a secondary port for the API
     # simcore_services fixture already ensure the dask-scheduler is up and running
     dask_scheduler_api_port = get_service_published_port(

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -4,75 +4,136 @@
 
 import asyncio
 import logging
-from typing import Dict, List
+from dataclasses import dataclass
+from typing import Dict, Final, List
 
 import aiohttp
 import pytest
 import tenacity
+from _pytest.monkeypatch import MonkeyPatch
+from aiohttp.client import ClientTimeout
+from tenacity.after import after_log
+from tenacity.before_sleep import before_sleep_log
+from tenacity.stop import stop_after_delay
+from tenacity.wait import wait_random
 from yarl import URL
 
-from .helpers.utils_docker import get_service_published_port
+from .helpers.utils_docker import get_ip, get_service_published_port
+from .helpers.utils_environs import EnvVarsDict
 
 log = logging.getLogger(__name__)
 
-SERVICES_TO_SKIP = ["dask-sidecar", "postgres", "redis", "rabbit"]
+SERVICES_TO_SKIP = [
+    "dask-sidecar",
+    "migration",
+    "postgres",
+    "redis",
+    "rabbit",
+    "static-webserver",
+    "whoami",
+    "traefik",
+]
+# TODO: unify healthcheck policies see  https://github.com/ITISFoundation/osparc-simcore/pull/2281
 SERVICE_PUBLISHED_PORT = {}
 SERVICE_HEALTHCHECK_ENTRYPOINT = {
     "director-v2": "/",
     "dask-scheduler": "/health",
+    "datcore-adapter": "/v0/live",
 }
 AIOHTTP_BASED_SERVICE_PORT: int = 8080
 FASTAPI_BASED_SERVICE_PORT: int = 8000
 DASK_SCHEDULER_SERVICE_PORT: int = 8787
 
 
+@dataclass
+class ServiceHealthcheckEndpoint:
+    name: str
+    url: URL
+
+    @classmethod
+    def create(cls, service_name: str, baseurl):
+        # TODO: unify healthcheck policies see  https://github.com/ITISFoundation/osparc-simcore/pull/2281
+        obj = cls(
+            name=service_name,
+            url=URL(
+                f"{baseurl}{SERVICE_HEALTHCHECK_ENTRYPOINT.get(service_name, '/v0/')}"
+            ),
+        )
+        return obj
+
+
 @pytest.fixture(scope="module")
 def services_endpoint(
-    core_services_selection: List[str], docker_stack: Dict, testing_environ_vars: Dict
+    core_services_selection: List[str],
+    docker_stack: Dict,
+    testing_environ_vars: EnvVarsDict,
 ) -> Dict[str, URL]:
     services_endpoint = {}
 
     stack_name = testing_environ_vars["SWARM_STACK_NAME"]
     for service in core_services_selection:
         assert f"{stack_name}_{service}" in docker_stack["services"]
+        full_service_name = f"{stack_name}_{service}"
+
+        # TODO: unify healthcheck policies see  https://github.com/ITISFoundation/osparc-simcore/pull/2281
         if service not in SERVICES_TO_SKIP:
             endpoint = URL(
-                f"http://127.0.0.1:{get_service_published_port(service, [AIOHTTP_BASED_SERVICE_PORT, FASTAPI_BASED_SERVICE_PORT, DASK_SCHEDULER_SERVICE_PORT])}"
+                f"http://{get_ip()}:{get_service_published_port(full_service_name, [AIOHTTP_BASED_SERVICE_PORT, FASTAPI_BASED_SERVICE_PORT, DASK_SCHEDULER_SERVICE_PORT])}"
             )
             services_endpoint[service] = endpoint
+        else:
+            print(f"Collecting service endpoints: '{service}' skipped")
+
     return services_endpoint
 
 
-@pytest.fixture(scope="function")
-async def simcore_services(services_endpoint: Dict[str, URL], monkeypatch) -> None:
+@pytest.fixture(scope="module")
+async def simcore_services_ready(
+    services_endpoint: Dict[str, URL], monkeypatch_module: MonkeyPatch
+) -> None:
 
-    # waits for all services to be responsive
-    wait_tasks = [
-        wait_till_service_responsive(
-            URL(f"{endpoint}{SERVICE_HEALTHCHECK_ENTRYPOINT.get(service, '/v0/')}")
-        )
-        for service, endpoint in services_endpoint.items()
+    # Compose and log healthcheck url entpoints
+
+    health_endpoints = [
+        ServiceHealthcheckEndpoint.create(service_name, endpoint)
+        for service_name, endpoint in services_endpoint.items()
     ]
-    await asyncio.gather(*wait_tasks, return_exceptions=False)
+
+    print("Composing health-check endpoints for relevant stack's services:")
+    for h in health_endpoints:
+        print(f" - {h.name} -> {h.url}")
+
+    # check ready
+    await asyncio.gather(
+        *[wait_till_service_responsive(h.name, h.url) for h in health_endpoints],
+        return_exceptions=False,
+    )
 
     # patches environment variables with host/port per service
     for service, endpoint in services_endpoint.items():
         env_prefix = service.upper().replace("-", "_")
-        monkeypatch.setenv(f"{env_prefix}_HOST", endpoint.host)
-        monkeypatch.setenv(f"{env_prefix}_PORT", str(endpoint.port))
+        assert endpoint.host
+        monkeypatch_module.setenv(f"{env_prefix}_HOST", endpoint.host)
+        monkeypatch_module.setenv(f"{env_prefix}_PORT", str(endpoint.port))
 
 
+_MINUTE: Final[int] = 60
 # HELPERS --
 @tenacity.retry(
-    wait=tenacity.wait_fixed(5),
-    stop=tenacity.stop_after_attempt(60),
-    before_sleep=tenacity.before_sleep_log(log, logging.INFO),
+    wait=wait_random(2, 15),
+    stop=stop_after_delay(5 * _MINUTE),
+    before_sleep=before_sleep_log(log, logging.WARNING),
+    after=after_log(log, logging.ERROR),
     reraise=True,
 )
-async def wait_till_service_responsive(endpoint: URL):
-    async with aiohttp.ClientSession() as session:
+async def wait_till_service_responsive(service_name: str, endpoint: URL):
+    print(f"trying to connect with '{service_name}' through '{endpoint}'")
+    async with aiohttp.ClientSession(timeout=ClientTimeout(total=1)) as session:
         async with session.get(endpoint) as resp:
             # NOTE: Health-check endpoint require only a
             # status code 200 (see e.g. services/web/server/docker/healthcheck.py)
             # regardless of the payload content
-            assert resp.status == 200
+            assert (
+                resp.status == 200
+            ), f"{service_name} NOT responsive on {endpoint}. Details: {resp}"
+            print(f"connection with {service_name} successful")

services/director-v2/src/simcore_service_director_v2/utils/logging_utils.py

@@ -117,7 +117,7 @@ def _log_arguments(
     }
 
     #  Before to the function execution, log function details.
-    logger_obj.info(
+    logger_obj.debug(
         "Arguments: %s - Begin function",
         formatted_arguments,
         extra=extra_args,

services/director-v2/tests/conftest.py

@@ -48,15 +48,16 @@
     "pytest_simcore.docker_registry",
     "pytest_simcore.docker_swarm",
     "pytest_simcore.environment_configs",
+    "pytest_simcore.monkeypatch_extra",
     "pytest_simcore.postgres_service",
     "pytest_simcore.pydantic_models",
     "pytest_simcore.rabbit_service",
     "pytest_simcore.redis_service",
     "pytest_simcore.repository_paths",
     "pytest_simcore.schemas",
+    "pytest_simcore.simcore_dask_service",
     "pytest_simcore.simcore_services",
     "pytest_simcore.tmp_path_extra",
-    "pytest_simcore.simcore_dask_service",
 ]
 
 logger = logging.getLogger(__name__)

services/director-v2/tests/integration/01/test_computation_api.py

@@ -33,7 +33,6 @@
 from simcore_service_director_v2.models.schemas.comp_tasks import ComputationTaskOut
 from starlette import status
 from starlette.testclient import TestClient
-from yarl import URL
 
 pytest_simcore_core_services_selection = [
     "director",
@@ -82,7 +81,7 @@ def minimal_configuration(
     postgres_db: sa.engine.Engine,
     postgres_host_config: Dict[str, str],
     rabbit_service: RabbitConfig,
-    simcore_services: Dict[str, URL],
+    simcore_services_ready: None,
 ) -> None:
     pass
 

services/director-v2/tests/integration/02/test__dynamic_sidecar_nodeports_integration.py

@@ -112,7 +112,7 @@ def minimal_configuration(  # pylint:disable=too-many-arguments
     postgres_db: sa.engine.Engine,
     postgres_host_config: Dict[str, str],
     rabbit_service: RabbitConfig,
-    simcore_services: None,
+    simcore_services_ready: None,
     dask_scheduler_service: None,
     dask_sidecar_service: None,
     ensure_swarm_and_networks: None,

services/director-v2/tests/integration/02/test_dynamic_services_routes.py

@@ -27,7 +27,7 @@
 @pytest.fixture
 def minimal_configuration(
     dy_static_file_server_dynamic_sidecar_service: Dict,
-    simcore_services: None,
+    simcore_services_ready: None,
 ):
     pass
 

services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py

@@ -58,7 +58,7 @@ def minimal_configuration(
     postgres_db: sa.engine.Engine,
     postgres_host_config: Dict[str, str],
     rabbit_service: RabbitConfig,
-    simcore_services: None,
+    simcore_services_ready: None,
     ensure_swarm_and_networks: None,
 ):
     pass

services/sidecar/tests/integration/test_mpi_lock.py

@@ -19,7 +19,7 @@ def redis_service_config(redis_service) -> None:
     mpi_lock.config.CELERY_CONFIG.redis = old_config
 
 
-async def test_mpi_locking(loop, simcore_services, redis_service_config) -> None:
+async def test_mpi_locking(loop, simcore_services_ready, redis_service_config) -> None:
     cpu_count = 2
 
     assert mpi_lock.acquire_mpi_lock(cpu_count) is True
@@ -28,7 +28,7 @@ async def test_mpi_locking(loop, simcore_services, redis_service_config) -> None
 
 @pytest.mark.parametrize("process_count, cpu_count", [(1, 3), (32, 4)])
 async def test_multiple_parallel_locking(
-    loop, simcore_services, redis_service_config, process_count, cpu_count
+    loop, simcore_services_ready, redis_service_config, process_count, cpu_count
 ) -> None:
     def worker(reply_queue: multiprocessing.Queue, cpu_count: int) -> None:
         mpi_lock_acquisition = mpi_lock.acquire_mpi_lock(cpu_count)

services/web/server/tests/integration/01/test_exporter.py

@@ -490,7 +490,7 @@ async def test_import_export_import_duplicate(
     aiopg_engine,
     redis_client,
     export_version,
-    simcore_services,
+    simcore_services_ready,
     monkey_patch_aiohttp_request_url,
     grant_access_rights,
 ):