diff --git a/ci/github/system-testing/public-api.bash b/ci/github/system-testing/public-api.bash index ca141a07ba6..491d59c652f 100755 --- a/ci/github/system-testing/public-api.bash +++ b/ci/github/system-testing/public-api.bash @@ -28,7 +28,12 @@ install() { } test() { - pytest --color=yes --cov-report=term-missing -v --keep-docker-up tests/public-api --log-level=DEBUG + pytest \ + --color=yes \ + --cov-report=term-missing \ + --keep-docker-up \ + -v \ + tests/public-api } clean_up() { diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index 91ca715b2b8..b0e8c19e094 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -13,7 +13,7 @@ from copy import deepcopy from pathlib import Path from pprint import pformat -from typing import Any, Dict, List, Union +from typing import Any, Dict, Iterator, List import pytest import yaml @@ -25,10 +25,11 @@ FIXTURE_CONFIG_OPS_SERVICES_SELECTION, ) from .helpers.utils_docker import get_ip, run_docker_compose_config, save_docker_infos +from .helpers.utils_environs import EnvVarsDict @pytest.fixture(scope="session") -def testing_environ_vars(env_devel_file: Path) -> Dict[str, Union[str, None]]: +def testing_environ_vars(env_devel_file: Path) -> EnvVarsDict: """ Loads and extends .env-devel returning all environment variables key=value @@ -36,7 +37,7 @@ def testing_environ_vars(env_devel_file: Path) -> Dict[str, Union[str, None]]: env_devel_unresolved = dotenv_values(env_devel_file, verbose=True, interpolate=True) # get from environ if applicable - env_devel = { + env_devel: EnvVarsDict = { key: os.environ.get(key, value) for key, value in env_devel_unresolved.items() } @@ -77,7 +78,7 @@ def env_file_for_testing( testing_environ_vars: Dict[str, str], temp_folder: Path, osparc_simcore_root_dir: Path, -) -> Path: +) -> Iterator[Path]: """Dumps all the environment variables into an $(temp_folder)/.env.test file Pass path as argument in 'docker-compose --env-file ... ' diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py index 83fb79ff615..d7a07e14890 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_swarm.py @@ -21,6 +21,7 @@ from tenacity.wait import wait_exponential, wait_fixed from .helpers.utils_docker import get_ip +from .helpers.utils_environs import EnvVarsDict log = logging.getLogger(__name__) @@ -140,13 +141,14 @@ def docker_stack( core_docker_compose_file: Path, ops_docker_compose_file: Path, keep_docker_up: bool, - testing_environ_vars: Dict, + testing_environ_vars: EnvVarsDict, ) -> Iterator[Dict]: # WARNING: keep prefix "pytest-" in stack names core_stack_name = testing_environ_vars["SWARM_STACK_NAME"] ops_stack_name = "pytest-ops" + assert core_stack_name assert core_stack_name.startswith("pytest-") stacks = [ ( diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py index 28c23010079..2d8dc076689 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/utils_environs.py @@ -4,10 +4,12 @@ import re from copy import deepcopy from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Union import yaml +EnvVarsDict = Dict[str, Union[str, None]] + VARIABLE_SUBSTITUTION = re.compile(r"\$\{(\w+)(?:(:{0,1}[-?]{0,1})(.*))?\}$") @@ -106,7 +108,7 @@ def eval_service_environ( image_environ: Dict = None, *, use_env_devel=True -) -> Dict: +) -> EnvVarsDict: """Deduces a service environment with it runs in a stack from confirmation :param docker_compose_path: path to stack configuration @@ -133,7 +135,7 @@ def eval_service_environ( image_environ = image_environ or {} # Environ expected in a running service - service_environ = {} + service_environ: EnvVarsDict = {} service_environ.update(image_environ) service_environ.update(service["environment"]) return service_environ diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py b/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py index 4a71b668e31..fd3ebbf8463 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_dask_service.py @@ -11,7 +11,7 @@ @pytest.fixture(scope="function") -async def dask_scheduler_service(simcore_services, monkeypatch) -> Dict[str, Any]: +async def dask_scheduler_service(simcore_services_ready, monkeypatch) -> Dict[str, Any]: # the dask scheduler has a UI for the dashboard and a secondary port for the API # simcore_services fixture already ensure the dask-scheduler is up and running dask_scheduler_api_port = get_service_published_port( diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index c8a9b669b7f..c6963cfb720 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -4,75 +4,136 @@ import asyncio import logging -from typing import Dict, List +from dataclasses import dataclass +from typing import Dict, Final, List import aiohttp import pytest import tenacity +from _pytest.monkeypatch import MonkeyPatch +from aiohttp.client import ClientTimeout +from tenacity.after import after_log +from tenacity.before_sleep import before_sleep_log +from tenacity.stop import stop_after_delay +from tenacity.wait import wait_random from yarl import URL -from .helpers.utils_docker import get_service_published_port +from .helpers.utils_docker import get_ip, get_service_published_port +from .helpers.utils_environs import EnvVarsDict log = logging.getLogger(__name__) -SERVICES_TO_SKIP = ["dask-sidecar", "postgres", "redis", "rabbit"] +SERVICES_TO_SKIP = [ + "dask-sidecar", + "migration", + "postgres", + "redis", + "rabbit", + "static-webserver", + "whoami", + "traefik", +] +# TODO: unify healthcheck policies see https://github.com/ITISFoundation/osparc-simcore/pull/2281 SERVICE_PUBLISHED_PORT = {} SERVICE_HEALTHCHECK_ENTRYPOINT = { "director-v2": "/", "dask-scheduler": "/health", + "datcore-adapter": "/v0/live", } AIOHTTP_BASED_SERVICE_PORT: int = 8080 FASTAPI_BASED_SERVICE_PORT: int = 8000 DASK_SCHEDULER_SERVICE_PORT: int = 8787 +@dataclass +class ServiceHealthcheckEndpoint: + name: str + url: URL + + @classmethod + def create(cls, service_name: str, baseurl): + # TODO: unify healthcheck policies see https://github.com/ITISFoundation/osparc-simcore/pull/2281 + obj = cls( + name=service_name, + url=URL( + f"{baseurl}{SERVICE_HEALTHCHECK_ENTRYPOINT.get(service_name, '/v0/')}" + ), + ) + return obj + + @pytest.fixture(scope="module") def services_endpoint( - core_services_selection: List[str], docker_stack: Dict, testing_environ_vars: Dict + core_services_selection: List[str], + docker_stack: Dict, + testing_environ_vars: EnvVarsDict, ) -> Dict[str, URL]: services_endpoint = {} stack_name = testing_environ_vars["SWARM_STACK_NAME"] for service in core_services_selection: assert f"{stack_name}_{service}" in docker_stack["services"] + full_service_name = f"{stack_name}_{service}" + + # TODO: unify healthcheck policies see https://github.com/ITISFoundation/osparc-simcore/pull/2281 if service not in SERVICES_TO_SKIP: endpoint = URL( - f"http://127.0.0.1:{get_service_published_port(service, [AIOHTTP_BASED_SERVICE_PORT, FASTAPI_BASED_SERVICE_PORT, DASK_SCHEDULER_SERVICE_PORT])}" + f"http://{get_ip()}:{get_service_published_port(full_service_name, [AIOHTTP_BASED_SERVICE_PORT, FASTAPI_BASED_SERVICE_PORT, DASK_SCHEDULER_SERVICE_PORT])}" ) services_endpoint[service] = endpoint + else: + print(f"Collecting service endpoints: '{service}' skipped") + return services_endpoint -@pytest.fixture(scope="function") -async def simcore_services(services_endpoint: Dict[str, URL], monkeypatch) -> None: +@pytest.fixture(scope="module") +async def simcore_services_ready( + services_endpoint: Dict[str, URL], monkeypatch_module: MonkeyPatch +) -> None: - # waits for all services to be responsive - wait_tasks = [ - wait_till_service_responsive( - URL(f"{endpoint}{SERVICE_HEALTHCHECK_ENTRYPOINT.get(service, '/v0/')}") - ) - for service, endpoint in services_endpoint.items() + # Compose and log healthcheck url entpoints + + health_endpoints = [ + ServiceHealthcheckEndpoint.create(service_name, endpoint) + for service_name, endpoint in services_endpoint.items() ] - await asyncio.gather(*wait_tasks, return_exceptions=False) + + print("Composing health-check endpoints for relevant stack's services:") + for h in health_endpoints: + print(f" - {h.name} -> {h.url}") + + # check ready + await asyncio.gather( + *[wait_till_service_responsive(h.name, h.url) for h in health_endpoints], + return_exceptions=False, + ) # patches environment variables with host/port per service for service, endpoint in services_endpoint.items(): env_prefix = service.upper().replace("-", "_") - monkeypatch.setenv(f"{env_prefix}_HOST", endpoint.host) - monkeypatch.setenv(f"{env_prefix}_PORT", str(endpoint.port)) + assert endpoint.host + monkeypatch_module.setenv(f"{env_prefix}_HOST", endpoint.host) + monkeypatch_module.setenv(f"{env_prefix}_PORT", str(endpoint.port)) +_MINUTE: Final[int] = 60 # HELPERS -- @tenacity.retry( - wait=tenacity.wait_fixed(5), - stop=tenacity.stop_after_attempt(60), - before_sleep=tenacity.before_sleep_log(log, logging.INFO), + wait=wait_random(2, 15), + stop=stop_after_delay(5 * _MINUTE), + before_sleep=before_sleep_log(log, logging.WARNING), + after=after_log(log, logging.ERROR), reraise=True, ) -async def wait_till_service_responsive(endpoint: URL): - async with aiohttp.ClientSession() as session: +async def wait_till_service_responsive(service_name: str, endpoint: URL): + print(f"trying to connect with '{service_name}' through '{endpoint}'") + async with aiohttp.ClientSession(timeout=ClientTimeout(total=1)) as session: async with session.get(endpoint) as resp: # NOTE: Health-check endpoint require only a # status code 200 (see e.g. services/web/server/docker/healthcheck.py) # regardless of the payload content - assert resp.status == 200 + assert ( + resp.status == 200 + ), f"{service_name} NOT responsive on {endpoint}. Details: {resp}" + print(f"connection with {service_name} successful") diff --git a/services/director-v2/src/simcore_service_director_v2/utils/logging_utils.py b/services/director-v2/src/simcore_service_director_v2/utils/logging_utils.py index 1317d9843d8..85a96bd7093 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/logging_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/logging_utils.py @@ -117,7 +117,7 @@ def _log_arguments( } # Before to the function execution, log function details. - logger_obj.info( + logger_obj.debug( "Arguments: %s - Begin function", formatted_arguments, extra=extra_args, diff --git a/services/director-v2/tests/conftest.py b/services/director-v2/tests/conftest.py index e7d4d320710..4f57b8e3947 100644 --- a/services/director-v2/tests/conftest.py +++ b/services/director-v2/tests/conftest.py @@ -48,15 +48,16 @@ "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", + "pytest_simcore.monkeypatch_extra", "pytest_simcore.postgres_service", "pytest_simcore.pydantic_models", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", "pytest_simcore.repository_paths", "pytest_simcore.schemas", + "pytest_simcore.simcore_dask_service", "pytest_simcore.simcore_services", "pytest_simcore.tmp_path_extra", - "pytest_simcore.simcore_dask_service", ] logger = logging.getLogger(__name__) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 75b884af0ed..81b6ed72f98 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -33,7 +33,6 @@ from simcore_service_director_v2.models.schemas.comp_tasks import ComputationTaskOut from starlette import status from starlette.testclient import TestClient -from yarl import URL pytest_simcore_core_services_selection = [ "director", @@ -82,7 +81,7 @@ def minimal_configuration( postgres_db: sa.engine.Engine, postgres_host_config: Dict[str, str], rabbit_service: RabbitConfig, - simcore_services: Dict[str, URL], + simcore_services_ready: None, ) -> None: pass diff --git a/services/director-v2/tests/integration/02/test__dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test__dynamic_sidecar_nodeports_integration.py index 6fcc6f117c5..1505c34795e 100644 --- a/services/director-v2/tests/integration/02/test__dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test__dynamic_sidecar_nodeports_integration.py @@ -112,7 +112,7 @@ def minimal_configuration( # pylint:disable=too-many-arguments postgres_db: sa.engine.Engine, postgres_host_config: Dict[str, str], rabbit_service: RabbitConfig, - simcore_services: None, + simcore_services_ready: None, dask_scheduler_service: None, dask_sidecar_service: None, ensure_swarm_and_networks: None, diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py index e900c87d8f7..318e2cb8b6e 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py @@ -27,7 +27,7 @@ @pytest.fixture def minimal_configuration( dy_static_file_server_dynamic_sidecar_service: Dict, - simcore_services: None, + simcore_services_ready: None, ): pass diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py index fb17f7d996b..ead37ce0f30 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -58,7 +58,7 @@ def minimal_configuration( postgres_db: sa.engine.Engine, postgres_host_config: Dict[str, str], rabbit_service: RabbitConfig, - simcore_services: None, + simcore_services_ready: None, ensure_swarm_and_networks: None, ): pass diff --git a/services/sidecar/tests/integration/test_mpi_lock.py b/services/sidecar/tests/integration/test_mpi_lock.py index ff83d7e4b8a..c8a60e4b7c0 100644 --- a/services/sidecar/tests/integration/test_mpi_lock.py +++ b/services/sidecar/tests/integration/test_mpi_lock.py @@ -19,7 +19,7 @@ def redis_service_config(redis_service) -> None: mpi_lock.config.CELERY_CONFIG.redis = old_config -async def test_mpi_locking(loop, simcore_services, redis_service_config) -> None: +async def test_mpi_locking(loop, simcore_services_ready, redis_service_config) -> None: cpu_count = 2 assert mpi_lock.acquire_mpi_lock(cpu_count) is True @@ -28,7 +28,7 @@ async def test_mpi_locking(loop, simcore_services, redis_service_config) -> None @pytest.mark.parametrize("process_count, cpu_count", [(1, 3), (32, 4)]) async def test_multiple_parallel_locking( - loop, simcore_services, redis_service_config, process_count, cpu_count + loop, simcore_services_ready, redis_service_config, process_count, cpu_count ) -> None: def worker(reply_queue: multiprocessing.Queue, cpu_count: int) -> None: mpi_lock_acquisition = mpi_lock.acquire_mpi_lock(cpu_count) diff --git a/services/web/server/tests/integration/01/test_exporter.py b/services/web/server/tests/integration/01/test_exporter.py index f42e6dc2c46..28bdfe51082 100644 --- a/services/web/server/tests/integration/01/test_exporter.py +++ b/services/web/server/tests/integration/01/test_exporter.py @@ -490,7 +490,7 @@ async def test_import_export_import_duplicate( aiopg_engine, redis_client, export_version, - simcore_services, + simcore_services_ready, monkey_patch_aiohttp_request_url, grant_access_rights, ): diff --git a/services/web/server/tests/integration/01/test_garbage_collection.py b/services/web/server/tests/integration/01/test_garbage_collection.py index 7e61f591447..b2029c71574 100644 --- a/services/web/server/tests/integration/01/test_garbage_collection.py +++ b/services/web/server/tests/integration/01/test_garbage_collection.py @@ -382,7 +382,7 @@ async def test_t1_while_guest_is_connected_no_resources_are_removed( async def test_t2_cleanup_resources_after_browser_is_closed( mock_garbage_collector_task, - simcore_services, + simcore_services_ready, client, socketio_client_factory: Callable, aiopg_engine, @@ -422,7 +422,7 @@ async def test_t2_cleanup_resources_after_browser_is_closed( async def test_t3_gc_will_not_intervene_for_regular_users_and_their_resources( - simcore_services, client, socketio_client_factory: Callable, aiopg_engine + simcore_services_ready, client, socketio_client_factory: Callable, aiopg_engine ): """after a USER disconnects the GC will remove none of its projects or templates nor the user itself""" number_of_projects = 5 @@ -463,7 +463,7 @@ async def assert_projects_and_users_are_present(): async def test_t4_project_shared_with_group_transferred_to_user_in_group_on_owner_removal( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a GROUP "g1" and invites USERS "u2" and "u3"; @@ -502,7 +502,7 @@ async def test_t4_project_shared_with_group_transferred_to_user_in_group_on_owne async def test_t5_project_shared_with_other_users_transferred_to_one_of_them( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a project and shares it with "u2" and "u3"; @@ -541,7 +541,7 @@ async def test_t5_project_shared_with_other_users_transferred_to_one_of_them( async def test_t6_project_shared_with_group_transferred_to_last_user_in_group_on_owner_removal( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a GROUP "g1" and invites USERS "u2" and "u3"; @@ -608,7 +608,7 @@ async def test_t6_project_shared_with_group_transferred_to_last_user_in_group_on async def test_t7_project_shared_with_group_transferred_from_one_member_to_the_last_and_all_is_removed( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a GROUP "g1" and invites USERS "u2" and "u3"; @@ -688,7 +688,7 @@ async def test_t7_project_shared_with_group_transferred_from_one_member_to_the_l async def test_t8_project_shared_with_other_users_transferred_to_one_of_them_until_one_user_remains( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a project and shares it with "u2" and "u3"; @@ -757,7 +757,7 @@ async def test_t8_project_shared_with_other_users_transferred_to_one_of_them_unt async def test_t9_project_shared_with_other_users_transferred_between_them_and_then_removed( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a project and shares it with "u2" and "u3"; @@ -839,7 +839,7 @@ async def test_t9_project_shared_with_other_users_transferred_between_them_and_t async def test_t10_owner_and_all_shared_users_marked_as_guests( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a project and shares it with "u2" and "u3"; @@ -879,7 +879,7 @@ async def test_t10_owner_and_all_shared_users_marked_as_guests( async def test_t11_owner_and_all_users_in_group_marked_as_guests( - simcore_services, client, aiopg_engine + simcore_services_ready, client, aiopg_engine ): """ USER "u1" creates a group and invites "u2" and "u3"; diff --git a/services/web/server/tests/integration/01/test_project_workflow.py b/services/web/server/tests/integration/01/test_project_workflow.py index 82bed42aff7..33afe7fd8b7 100644 --- a/services/web/server/tests/integration/01/test_project_workflow.py +++ b/services/web/server/tests/integration/01/test_project_workflow.py @@ -199,7 +199,7 @@ async def _request_delete(client, pid): async def test_workflow( postgres_db: sa.engine.Engine, docker_registry: str, - simcore_services, + simcore_services_ready, fake_project_data, catalog_subsystem_mock, client, @@ -288,7 +288,7 @@ async def test_get_invalid_project( client, postgres_db: sa.engine.Engine, docker_registry: str, - simcore_services, + simcore_services_ready, logged_user, ): url = client.app.router["get_project"].url_for(project_id="some-fake-id") @@ -302,7 +302,7 @@ async def test_update_invalid_project( client, postgres_db: sa.engine.Engine, docker_registry: str, - simcore_services, + simcore_services_ready, logged_user, ): url = client.app.router["replace_project"].url_for(project_id="some-fake-id") @@ -316,7 +316,7 @@ async def test_delete_invalid_project( client, postgres_db: sa.engine.Engine, docker_registry: str, - simcore_services, + simcore_services_ready, logged_user, ): url = client.app.router["delete_project"].url_for(project_id="some-fake-id") diff --git a/services/web/server/tests/integration/02/test_computation.py b/services/web/server/tests/integration/02/test_computation.py index e6b1b98d9d9..b037edf765a 100644 --- a/services/web/server/tests/integration/02/test_computation.py +++ b/services/web/server/tests/integration/02/test_computation.py @@ -229,7 +229,7 @@ async def test_start_pipeline( postgres_session: sa.orm.session.Session, rabbit_service: RabbitConfig, redis_service: RedisConfig, - simcore_services: Dict[str, URL], + simcore_services_ready: Dict[str, URL], client, socketio_client_factory: Callable, logged_user: LoggedUser, diff --git a/services/web/server/tests/integration/conftest.py b/services/web/server/tests/integration/conftest.py index 59ed6eb8493..ec01bf9e347 100644 --- a/services/web/server/tests/integration/conftest.py +++ b/services/web/server/tests/integration/conftest.py @@ -45,6 +45,7 @@ "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", + "pytest_simcore.monkeypatch_extra", "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", diff --git a/services/web/server/tests/integration/webserver_dev_config.yaml b/services/web/server/tests/integration/webserver_dev_config.yaml new file mode 100644 index 00000000000..bdb2f7492be --- /dev/null +++ b/services/web/server/tests/integration/webserver_dev_config.yaml @@ -0,0 +1,63 @@ +activity: + enabled: true + prometheus_api_version: ${WEBSERVER_PROMETHEUS_API_VERSION} + prometheus_host: ${WEBSERVER_PROMETHEUS_HOST} + prometheus_port: ${WEBSERVER_PROMETHEUS_PORT} +catalog: + host: ${CATALOG_HOST} + port: ${CATALOG_PORT} + version: v0 +db: + postgres: + database: ${POSTGRES_DB} + endpoint: ${POSTGRES_ENDPOINT} + host: ${POSTGRES_HOST} + maxsize: 50 + minsize: 1 + password: ${POSTGRES_PASSWORD} + port: ${POSTGRES_PORT} + user: ${POSTGRES_USER} +director: + host: ${DIRECTOR_HOST} + port: ${DIRECTOR_PORT} + version: v0 +login: + enabled: true + registration_confirmation_required: false + registration_invitation_required: false +main: + host: 127.0.0.1 + log_level: ${WEBSERVER_LOGLEVEL} + port: 8080 + studies_access_enabled: true + testing: true +projects: + enabled: true +resource_manager: + enabled: true + garbage_collection_interval_seconds: ${WEBSERVER_GARBAGE_COLLECTION_INTERVAL_SECONDS} + redis: + host: ${REDIS_HOST} + port: ${REDIS_PORT} + resource_deletion_timeout_seconds: ${WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS} +rest: + version: v0 +session: + secret_key: ${WEBSERVER_SESSION_SECRET_KEY} +smtp: + host: ${SMTP_HOST} + password: null + port: ${SMTP_PORT} + sender: OSPARC support + tls: false + username: null +socketio: + enabled: true +storage: + host: ${STORAGE_HOST} + port: ${STORAGE_PORT} + version: v0 +tracing: + enabled: true + zipkin_endpoint: ${TRACING_ZIPKIN_ENDPOINT} +version: '1.0' diff --git a/tests/public-api/conftest.py b/tests/public-api/conftest.py index 58be758169e..6cd61dcbefe 100644 --- a/tests/public-api/conftest.py +++ b/tests/public-api/conftest.py @@ -2,17 +2,21 @@ # pylint:disable=unused-argument # pylint:disable=redefined-outer-name +import asyncio import logging import os import time from pprint import pformat -from typing import Any, Callable, Dict, List +from typing import Any, Callable, Dict, Iterable, List import httpx import osparc import pytest from osparc.configuration import Configuration -from tenacity import Retrying, before_sleep_log, stop_after_attempt, wait_fixed +from tenacity.before_sleep import before_sleep_log +from tenacity.wait import wait_fixed +from tenacity.stop import stop_after_attempt +from tenacity import Retrying log = logging.getLogger(__name__) @@ -21,8 +25,10 @@ "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", + "pytest_simcore.monkeypatch_extra", "pytest_simcore.repository_paths", "pytest_simcore.schemas", + "pytest_simcore.simcore_services", "pytest_simcore.tmp_path_extra", ] @@ -52,10 +58,20 @@ def ops_services_selection(ops_docker_compose: Dict) -> List[str]: return all_ops_services +@pytest.fixture(scope="module") +def event_loop(request) -> Iterable[asyncio.AbstractEventLoop]: + """Overrides pytest_asyncio.event_loop and extends to module scope""" + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + @pytest.fixture(scope="module") def simcore_docker_stack_and_registry_ready( + event_loop: asyncio.AbstractEventLoop, docker_stack: Dict, docker_registry, + simcore_services_ready: None, ) -> Dict: for attempt in Retrying( diff --git a/tests/public-api/requirements/_test.in b/tests/public-api/requirements/_test.in index 18038ace6db..b0693f96b52 100644 --- a/tests/public-api/requirements/_test.in +++ b/tests/public-api/requirements/_test.in @@ -5,11 +5,13 @@ pytest-cov pytest-asyncio faker + python-dotenv httpx # pulled by pytest-simcore +aiohttp docker -tenacity jsonschema pyyaml +tenacity diff --git a/tests/public-api/requirements/_test.txt b/tests/public-api/requirements/_test.txt index b691698582e..65c56e93d80 100644 --- a/tests/public-api/requirements/_test.txt +++ b/tests/public-api/requirements/_test.txt @@ -4,10 +4,19 @@ # # pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in # +aiohttp==3.8.0 + # via + # -c requirements/../../../requirements/constraints.txt + # -r requirements/_test.in +aiosignal==1.2.0 + # via aiohttp anyio==3.3.1 # via httpcore +async-timeout==4.0.0a3 + # via aiohttp attrs==21.2.0 # via + # aiohttp # jsonschema # pytest certifi==2021.5.30 @@ -16,6 +25,7 @@ certifi==2021.5.30 # requests charset-normalizer==2.0.6 # via + # aiohttp # httpx # requests coverage==5.5 @@ -24,6 +34,10 @@ docker==5.0.2 # via -r requirements/_test.in faker==8.13.2 # via -r requirements/_test.in +frozenlist==1.2.0 + # via + # aiohttp + # aiosignal h11==0.12.0 # via httpcore httpcore==0.13.7 @@ -35,10 +49,15 @@ idna==3.2 # anyio # requests # rfc3986 + # yarl iniconfig==1.1.1 # via pytest jsonschema==3.2.0 # via -r requirements/_test.in +multidict==5.2.0 + # via + # aiohttp + # yarl packaging==21.0 # via pytest pluggy==1.0.0 @@ -87,12 +106,16 @@ toml==0.10.2 # via # pytest # pytest-cov +typing-extensions==3.10.0.2 + # via async-timeout urllib3==1.26.6 # via # -c requirements/../../../requirements/constraints.txt # requests websocket-client==1.2.1 # via docker +yarl==1.7.0 + # via aiohttp # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/tests/public-api/requirements/_tools.txt b/tests/public-api/requirements/_tools.txt index 8eed87e8639..81a26178a9d 100644 --- a/tests/public-api/requirements/_tools.txt +++ b/tests/public-api/requirements/_tools.txt @@ -60,7 +60,9 @@ tomli==1.2.1 # black # pep517 typing-extensions==3.10.0.2 - # via black + # via + # -c requirements/_test.txt + # black virtualenv==20.8.0 # via pre-commit wheel==0.37.0