From 224cfd7bdb3b536d2ad3c568e1900bfbcefa7643 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Thu, 13 Apr 2023 13:49:30 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Metrics:=20add=20simcore=5Fuser=5Fa?= =?UTF-8?q?gent=20in=20service=5Fstarted/service=5Fstopped=20metric=20(#40?= =?UTF-8?q?92)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/models_library/rabbitmq_messages.py | 5 +- .../servicelib/aiohttp/monitor_services.py | 14 ++--- .../src/servicelib/aiohttp/monitoring.py | 29 +++++---- .../src/servicelib/common_headers.py | 1 + .../tests/aiohttp/test_monitoring.py | 11 ++-- .../modules/comp_scheduler/dask_scheduler.py | 3 + .../scheduler/_core/_events.py | 1 + .../scheduler/_core/_events_utils.py | 1 + .../director_v2_core_dynamic_services.py | 24 ++++---- .../exporter/formatters/formatter_v1.py | 16 +++-- .../garbage_collector_core.py | 20 ++++++- .../projects/_delete.py | 17 ++++-- .../projects/projects_api.py | 60 +++++++++++++------ .../projects/projects_handlers.py | 13 +++- .../projects/projects_handlers_crud.py | 30 +++++++--- .../projects/projects_nodes_handlers.py | 12 +++- .../tests/integration/02/test_rabbit.py | 2 + .../test_director_v2_core_dynamic_services.py | 11 +++- .../test_studies_dispatcher_studies_access.py | 30 ++++++---- .../02/test_projects_handlers__delete.py | 2 + .../02/test_projects_handlers__open_close.py | 16 ++--- .../02/test_projects_nodes_handler.py | 8 ++- .../test_resource_manager.py | 13 +++- 23 files changed, 233 insertions(+), 106 deletions(-) diff --git a/packages/models-library/src/models_library/rabbitmq_messages.py b/packages/models-library/src/models_library/rabbitmq_messages.py index e4cda73e93a..ad40c6e4b53 100644 --- a/packages/models-library/src/models_library/rabbitmq_messages.py +++ b/packages/models-library/src/models_library/rabbitmq_messages.py @@ -1,6 +1,6 @@ import logging from enum import Enum, auto -from typing import Any, Literal, Optional +from typing import Any, Literal from models_library.projects import ProjectID from models_library.projects_nodes import NodeID @@ -85,7 +85,8 @@ class InstrumentationRabbitMessage(RabbitMessageBase, NodeMessageBase): service_type: str service_key: str service_tag: str - result: Optional[RunningState] = None + result: RunningState | None = None + simcore_user_agent: str class _RabbitAutoscalingBaseMessage(RabbitMessageBase): diff --git a/packages/service-library/src/servicelib/aiohttp/monitor_services.py b/packages/service-library/src/servicelib/aiohttp/monitor_services.py index 68a02dcc512..3f50b7320d4 100644 --- a/packages/service-library/src/servicelib/aiohttp/monitor_services.py +++ b/packages/service-library/src/servicelib/aiohttp/monitor_services.py @@ -1,5 +1,4 @@ from enum import Enum -from typing import Union from aiohttp import web from prometheus_client import Counter @@ -26,22 +25,19 @@ kSERVICE_STARTED = f"{__name__}.services_started" kSERVICE_STOPPED = f"{__name__}.services_stopped" -SERVICE_STARTED_LABELS: list[str] = [ - "service_key", - "service_tag", -] +SERVICE_STARTED_LABELS: list[str] = ["service_key", "service_tag", "simcore_user_agent"] SERVICE_STOPPED_LABELS: list[str] = [ "service_key", "service_tag", "result", + "simcore_user_agent", ] def add_instrumentation( app: web.Application, reg: CollectorRegistry, app_name: str ) -> None: - app[kSERVICE_STARTED] = Counter( name="services_started_total", documentation="Counts the services started", @@ -71,10 +67,12 @@ def service_started( app: web.Application, service_key: str, service_tag: str, + simcore_user_agent: str, ) -> None: app[kSERVICE_STARTED].labels( service_key=service_key, service_tag=service_tag, + simcore_user_agent=simcore_user_agent, ).inc() @@ -83,10 +81,12 @@ def service_stopped( app: web.Application, service_key: str, service_tag: str, - result: Union[ServiceResult, str], + simcore_user_agent: str, + result: ServiceResult | str, ) -> None: app[kSERVICE_STOPPED].labels( service_key=service_key, service_tag=service_tag, + simcore_user_agent=simcore_user_agent, result=result.name if isinstance(result, ServiceResult) else result, ).inc() diff --git a/packages/service-library/src/servicelib/aiohttp/monitoring.py b/packages/service-library/src/servicelib/aiohttp/monitoring.py index 7bbed9c196b..5f975bbc2d3 100644 --- a/packages/service-library/src/servicelib/aiohttp/monitoring.py +++ b/packages/service-library/src/servicelib/aiohttp/monitoring.py @@ -5,7 +5,7 @@ import asyncio import logging import time -from typing import Awaitable, Callable, Final, Optional +from typing import Awaitable, Callable import prometheus_client from aiohttp import web @@ -21,7 +21,10 @@ from prometheus_client.registry import CollectorRegistry from servicelib.aiohttp.typing_extension import Handler -from ..common_headers import X_SIMCORE_USER_AGENT +from ..common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_SIMCORE_USER_AGENT, +) from ..logging_utils import log_catch log = logging.getLogger(__name__) @@ -113,8 +116,6 @@ kPLATFORM_COLLECTOR = f"{__name__}.collector_platform" kGC_COLLECTOR = f"{__name__}.collector_gc" -UNDEFINED_REGULAR_USER_AGENT: Final[str] = "undefined" - def get_collector_registry(app: web.Application) -> CollectorRegistry: return app[kCOLLECTOR_REGISTRY] @@ -138,8 +139,8 @@ async def metrics_handler(request: web.Request): def middleware_factory( app_name: str, - enter_middleware_cb: Optional[EnterMiddlewareCB], - exit_middleware_cb: Optional[ExitMiddlewareCB], + enter_middleware_cb: EnterMiddlewareCB | None, + exit_middleware_cb: ExitMiddlewareCB | None, ): @web.middleware async def middleware_handler(request: web.Request, handler: Handler): @@ -168,12 +169,16 @@ async def middleware_handler(request: web.Request, handler: Handler): app_name, request.method, canonical_endpoint, - request.headers.get(X_SIMCORE_USER_AGENT, UNDEFINED_REGULAR_USER_AGENT), + request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), ).track_inprogress(), response_summary.labels( app_name, request.method, canonical_endpoint, - request.headers.get(X_SIMCORE_USER_AGENT, UNDEFINED_REGULAR_USER_AGENT), + request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), ).time(): resp = await handler(request) @@ -209,7 +214,9 @@ async def middleware_handler(request: web.Request, handler: Handler): request.method, canonical_endpoint, resp.status, - request.headers.get(X_SIMCORE_USER_AGENT, UNDEFINED_REGULAR_USER_AGENT), + request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), ).inc() if exit_middleware_cb: @@ -242,8 +249,8 @@ def setup_monitoring( app: web.Application, app_name: str, *, - enter_middleware_cb: Optional[EnterMiddlewareCB] = None, - exit_middleware_cb: Optional[ExitMiddlewareCB] = None, + enter_middleware_cb: EnterMiddlewareCB | None = None, + exit_middleware_cb: ExitMiddlewareCB | None = None, **app_info_kwargs, ): # app-scope registry diff --git a/packages/service-library/src/servicelib/common_headers.py b/packages/service-library/src/servicelib/common_headers.py index 842e9d14986..f883a99b9b4 100644 --- a/packages/service-library/src/servicelib/common_headers.py +++ b/packages/service-library/src/servicelib/common_headers.py @@ -4,3 +4,4 @@ X_DYNAMIC_SIDECAR_REQUEST_SCHEME: Final[str] = "X-Dynamic-Sidecar-Request-Scheme" X_FORWARDED_PROTO: Final[str] = "X-Forwarded-Proto" X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent" +UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE: Final[str] = "undefined" diff --git a/packages/service-library/tests/aiohttp/test_monitoring.py b/packages/service-library/tests/aiohttp/test_monitoring.py index 18b6043b658..74241c5ff26 100644 --- a/packages/service-library/tests/aiohttp/test_monitoring.py +++ b/packages/service-library/tests/aiohttp/test_monitoring.py @@ -11,8 +11,11 @@ from aiohttp.test_utils import TestClient from faker import Faker from prometheus_client.parser import text_string_to_metric_families -from servicelib.aiohttp.monitoring import UNDEFINED_REGULAR_USER_AGENT, setup_monitoring -from servicelib.common_headers import X_SIMCORE_USER_AGENT +from servicelib.aiohttp.monitoring import setup_monitoring +from servicelib.common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_SIMCORE_USER_AGENT, +) @pytest.fixture @@ -93,7 +96,7 @@ async def test_setup_monitoring(client: TestClient): "endpoint": "/monitored_request", "http_status": "200", "method": "GET", - "simcore_user_agent": UNDEFINED_REGULAR_USER_AGENT, + "simcore_user_agent": UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, }, value=NUM_CALLS, ) @@ -107,7 +110,7 @@ async def test_setup_monitoring(client: TestClient): "endpoint": "/metrics", "http_status": "200", "method": "GET", - "simcore_user_agent": UNDEFINED_REGULAR_USER_AGENT, + "simcore_user_agent": UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, }, value=1, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py index 32ef1037661..71d051a23ed 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/comp_scheduler/dask_scheduler.py @@ -21,6 +21,7 @@ ProgressRabbitMessageNode, ) from models_library.users import UserID +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.comp_tasks import NodeClass from simcore_service_director_v2.core.errors import TaskSchedulingError @@ -198,6 +199,7 @@ async def _process_task_result( service_key=service_key, service_tag=service_version, result=task_final_state, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, ) await self.rabbitmq_client.publish(message.channel_name, message.json()) @@ -225,6 +227,7 @@ async def _task_state_change_handler(self, event: str) -> None: service_type=NodeClass.COMPUTATIONAL.value, service_key=service_key, service_tag=service_version, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, ) await self.rabbitmq_client.publish(message.channel_name, message.json()) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py index 8d9fb52aaaa..8982876ef51 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py @@ -116,6 +116,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: service_type=NodeClass.INTERACTIVE.value, service_key=scheduler_data.key, service_tag=scheduler_data.version, + simcore_user_agent=scheduler_data.request_simcore_user_agent, ) rabbitmq_client: RabbitMQClient = app.state.rabbitmq_client await rabbitmq_client.publish(message.channel_name, message.json()) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index af4c096bf35..9d3995670b0 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -337,6 +337,7 @@ async def _remove_containers_save_state_and_outputs() -> None: service_type=NodeClass.INTERACTIVE.value, service_key=scheduler_data.key, service_tag=scheduler_data.version, + simcore_user_agent=scheduler_data.request_simcore_user_agent, ) rabbitmq_client: RabbitMQClient = app.state.rabbitmq_client await rabbitmq_client.publish(message.channel_name, message.json()) diff --git a/services/web/server/src/simcore_service_webserver/director_v2_core_dynamic_services.py b/services/web/server/src/simcore_service_webserver/director_v2_core_dynamic_services.py index 842c9a1974c..9ae99883629 100644 --- a/services/web/server/src/simcore_service_webserver/director_v2_core_dynamic_services.py +++ b/services/web/server/src/simcore_service_webserver/director_v2_core_dynamic_services.py @@ -7,7 +7,6 @@ import logging from contextlib import AsyncExitStack from functools import partial -from typing import Optional from aiohttp import web from models_library.projects import ProjectID @@ -40,11 +39,10 @@ log = logging.getLogger(__name__) -@log_decorator(logger=log) async def list_dynamic_services( app: web.Application, - user_id: Optional[PositiveInt] = None, - project_id: Optional[str] = None, + user_id: PositiveInt | None = None, + project_id: str | None = None, ) -> list[DataType]: params = {} if user_id: @@ -80,7 +78,6 @@ async def get_dynamic_service(app: web.Application, node_uuid: str) -> DataType: return service_state -@log_decorator(logger=log) async def run_dynamic_service( *, app: web.Application, @@ -92,7 +89,7 @@ async def run_dynamic_service( service_uuid: str, request_dns: str, request_scheme: str, - request_simcore_user_agent: str, + simcore_user_agent: str, service_resources: ServiceResourcesDict, ) -> DataType: """ @@ -116,7 +113,7 @@ async def run_dynamic_service( headers = { X_DYNAMIC_SIDECAR_REQUEST_DNS: request_dns, X_DYNAMIC_SIDECAR_REQUEST_SCHEME: request_scheme, - X_SIMCORE_USER_AGENT: request_simcore_user_agent, + X_SIMCORE_USER_AGENT: simcore_user_agent, } settings: DirectorV2Settings = get_plugin_settings(app) @@ -133,18 +130,21 @@ async def run_dynamic_service( return started_service -@log_decorator(logger=log) async def stop_dynamic_service( app: web.Application, service_uuid: NodeIDStr, + simcore_user_agent: str, save_state: bool = True, - progress: Optional[ProgressBarData] = None, + progress: ProgressBarData | None = None, ) -> None: """ Stopping a service can take a lot of time bumping the stop command timeout to 1 hour this will allow to sava bigger datasets from the services """ + headers = { + X_SIMCORE_USER_AGENT: simcore_user_agent, + } settings: DirectorV2Settings = get_plugin_settings(app) async with AsyncExitStack() as stack: @@ -157,6 +157,7 @@ async def stop_dynamic_service( url=(settings.base_url / f"dynamic_services/{service_uuid}").update_query( can_save="true" if save_state else "false", ), + headers=headers, expected_status=web.HTTPNoContent, timeout=settings.DIRECTOR_V2_STOP_SERVICE_TIMEOUT, on_error={ @@ -176,7 +177,7 @@ async def _post_progress_message( ) -> None: progress_message = ProgressRabbitMessageProject( user_id=user_id, - project_id=project_id, + project_id=ProjectID(project_id), progress_type=ProgressType.PROJECT_CLOSING, progress=progress_value, ) @@ -186,11 +187,11 @@ async def _post_progress_message( ) -@log_decorator(logger=log) async def stop_dynamic_services_in_project( app: web.Application, user_id: PositiveInt, project_id: str, + simcore_user_agent: str, save_state: bool = True, ) -> None: """Stops all dynamic services of either project_id or user_id in concurrently""" @@ -215,6 +216,7 @@ async def stop_dynamic_services_in_project( stop_dynamic_service( app=app, service_uuid=service["service_uuid"], + simcore_user_agent=simcore_user_agent, save_state=save_state, progress=progress_bar.sub_progress(1), ) diff --git a/services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v1.py b/services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v1.py index 85db265216e..4ac21852bbb 100644 --- a/services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v1.py +++ b/services/web/server/src/simcore_service_webserver/exporter/formatters/formatter_v1.py @@ -5,7 +5,7 @@ from collections import deque from itertools import chain from pathlib import Path -from typing import Deque, Optional +from typing import Deque from uuid import UUID from aiohttp import ClientError, ClientSession, ClientTimeout, web @@ -22,6 +22,7 @@ from models_library.utils.nodes import compute_node_hash, project_node_io_payload_cb from pydantic import AnyUrl, parse_obj_as from servicelib.aiohttp.client_session import get_client_session +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from servicelib.utils import logged_gather from simcore_sdk.node_ports_common.exceptions import ( NodeportsException, @@ -141,7 +142,7 @@ async def extract_download_links( async def generate_directory_contents( app: web.Application, root_folder: Path, - manifest_root_folder: Optional[Path], + manifest_root_folder: Path | None, project_id: str, user_id: int, version: str, @@ -348,7 +349,7 @@ async def import_files_and_validate_project( user_id: int, product_name: str, root_folder: Path, - manifest_root_folder: Optional[Path], + manifest_root_folder: Path | None, ) -> str: project_file = await ProjectFile.model_from_file(root_dir=root_folder) shuffled_data: ShuffledData = project_file.get_shuffled_uuids() @@ -420,7 +421,10 @@ async def import_files_and_validate_project( ) try: await submit_delete_project_task( - app=app, project_uuid=UUID(project_uuid), user_id=user_id + app=app, + project_uuid=UUID(project_uuid), + user_id=user_id, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, ) except ProjectsException: # no need to raise an error here @@ -440,7 +444,7 @@ async def format_export_directory( self, app: web.Application, project_id: str, user_id: int, **kwargs ) -> None: # injected by Formatter_V2 - manifest_root_folder: Optional[Path] = kwargs.get("manifest_root_folder") + manifest_root_folder: Path | None = kwargs.get("manifest_root_folder") await generate_directory_contents( app=app, @@ -456,7 +460,7 @@ async def validate_and_import_directory(self, **kwargs) -> str: user_id: int = kwargs["user_id"] product_name: str = kwargs["product_name"] # injected by Formatter_V2 - manifest_root_folder: Optional[Path] = kwargs.get("manifest_root_folder") + manifest_root_folder: Path | None = kwargs.get("manifest_root_folder") return await import_files_and_validate_project( app=app, diff --git a/services/web/server/src/simcore_service_webserver/garbage_collector_core.py b/services/web/server/src/simcore_service_webserver/garbage_collector_core.py index 45f76b04730..7ff3c7cd153 100644 --- a/services/web/server/src/simcore_service_webserver/garbage_collector_core.py +++ b/services/web/server/src/simcore_service_webserver/garbage_collector_core.py @@ -9,7 +9,9 @@ import asyncpg.exceptions from aiohttp import web +from models_library.projects import ProjectID from redis.asyncio import Redis +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from servicelib.logging_utils import log_context, log_decorator from servicelib.utils import logged_gather from simcore_postgres_database.errors import DatabaseError @@ -196,6 +198,7 @@ async def remove_disconnected_user_resources( user_id=int(dead_key["user_id"]), project_uuid=resource_value, app=app, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, user_name={ "first_name": "garbage", "last_name": "collector", @@ -321,7 +324,10 @@ async def _remove_single_service_if_orphan( ) try: await director_v2_api.stop_dynamic_service( - app, service_uuid, save_state=False + app, + service_uuid, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + save_state=False, ) except (ServiceNotFoundError, DirectorException) as err: logger.warning("Error while stopping service: %s", err) @@ -378,7 +384,10 @@ async def _remove_single_service_if_orphan( try: await director_v2_api.stop_dynamic_service( - app, service_uuid, save_state + app, + service_uuid, + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + save_state, ) except ServiceWaitingForManualIntervention: pass @@ -518,7 +527,12 @@ async def _delete_all_projects_for_user(app: web.Application, user_id: int) -> N f"{project_uuid=}", f"{user_id=}", ) - task = await submit_delete_project_task(app, project_uuid, user_id) + task = await submit_delete_project_task( + app, + ProjectID(project_uuid), + user_id, + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + ) assert task # nosec delete_tasks.append(task) diff --git a/services/web/server/src/simcore_service_webserver/projects/_delete.py b/services/web/server/src/simcore_service_webserver/projects/_delete.py index 19354d4c30b..00891c79f4f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/_delete.py +++ b/services/web/server/src/simcore_service_webserver/projects/_delete.py @@ -7,7 +7,7 @@ import asyncio import logging -from typing import Optional, Protocol +from typing import Protocol from aiohttp import web from models_library.projects import ProjectID @@ -37,8 +37,9 @@ async def __call__( user_id: int, project_uuid: str, app: web.Application, + simcore_user_agent: str, notify_users: bool = True, - user_name: Optional[UserNameDict] = None, + user_name: UserNameDict | None = None, ) -> None: ... @@ -68,6 +69,7 @@ async def delete_project( app: web.Application, project_uuid: ProjectID, user_id: UserID, + simcore_user_agent, # TODO: this function was tmp added here to avoid refactoring all projects_api in a single PR remove_project_dynamic_services: RemoveProjectServicesCallable, ) -> None: @@ -91,7 +93,7 @@ async def delete_project( # stops dynamic services # - raises ProjectNotFoundError, UserNotFoundError, ProjectLockError await remove_project_dynamic_services( - user_id, f"{project_uuid}", app, notify_users=False + user_id, f"{project_uuid}", app, simcore_user_agent, notify_users=False ) # stops computational services @@ -119,6 +121,7 @@ def schedule_task( app: web.Application, project_uuid: ProjectID, user_id: UserID, + simcore_user_agent: str, remove_project_dynamic_services: RemoveProjectServicesCallable, logger: logging.Logger, ) -> asyncio.Task: @@ -161,7 +164,13 @@ def _log_state_when_done(fut: asyncio.Future): # ------ task = asyncio.create_task( - delete_project(app, project_uuid, user_id, remove_project_dynamic_services), + delete_project( + app, + project_uuid, + user_id, + simcore_user_agent, + remove_project_dynamic_services, + ), name=DELETE_PROJECT_TASK_NAME.format(project_uuid, user_id), ) diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_api.py b/services/web/server/src/simcore_service_webserver/projects/projects_api.py index 89508389228..e6d86797460 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_api.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_api.py @@ -15,7 +15,7 @@ from contextlib import suppress from datetime import datetime from pprint import pformat -from typing import Any, Optional +from typing import Any from uuid import UUID, uuid4 from aiohttp import web @@ -40,7 +40,11 @@ APP_JSONSCHEMA_SPECS_KEY, ) from servicelib.aiohttp.jsonschema_validation import validate_instance -from servicelib.common_headers import X_FORWARDED_PROTO, X_SIMCORE_USER_AGENT +from servicelib.common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_FORWARDED_PROTO, + X_SIMCORE_USER_AGENT, +) from servicelib.json_serialization import json_dumps from servicelib.logging_utils import log_context from servicelib.utils import fire_and_forget_task, logged_gather @@ -105,7 +109,7 @@ async def get_project_for_user( project_uuid: str, user_id: UserID, *, - include_state: Optional[bool] = False, + include_state: bool | None = False, check_permissions: str = "read", ) -> dict: """Returns a VALID project accessible to user @@ -159,7 +163,10 @@ async def update_project_last_change_timestamp( async def submit_delete_project_task( - app: web.Application, project_uuid: ProjectID, user_id: UserID + app: web.Application, + project_uuid: ProjectID, + user_id: UserID, + simcore_user_agent: str, ) -> asyncio.Task: """ Marks a project as deleted and schedules a task to performe the entire removal workflow @@ -180,14 +187,19 @@ async def submit_delete_project_task( task = get_delete_project_task(project_uuid, user_id) if not task: task = _delete.schedule_task( - app, project_uuid, user_id, remove_project_dynamic_services, log + app, + project_uuid, + user_id, + simcore_user_agent, + remove_project_dynamic_services, + log, ) return task def get_delete_project_task( project_uuid: ProjectID, user_id: UserID -) -> Optional[asyncio.Task]: +) -> asyncio.Task | None: if tasks := _delete.get_scheduled_tasks(project_uuid, user_id): assert len(tasks) == 1, f"{tasks=}" # nosec task = tasks[0] @@ -248,7 +260,9 @@ async def _start_dynamic_service( service_uuid=f"{node_uuid}", request_dns=extract_dns_without_default_port(request.url), request_scheme=request.headers.get(X_FORWARDED_PROTO, request.url.scheme), - request_simcore_user_agent=request.headers.get(X_SIMCORE_USER_AGENT, ""), + simcore_user_agent=request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), service_resources=service_resources, ) @@ -260,7 +274,7 @@ async def add_project_node( product_name: str, service_key: str, service_version: str, - service_id: Optional[str], + service_id: str | None, ) -> str: log.debug( "starting node %s:%s in project %s for user %s", @@ -336,7 +350,7 @@ async def start_project_node( async def delete_project_node( - request: web.Request, project_uuid: ProjectID, user_id: UserID, node_uuid: str + request: web.Request, project_uuid: ProjectID, user_id: UserID, node_uuid: NodeIDStr ) -> None: log.debug( "deleting node %s in project %s for user %s", node_uuid, project_uuid, user_id @@ -350,6 +364,9 @@ async def delete_project_node( await director_v2_api.stop_dynamic_service( request.app, node_uuid, + simcore_user_agent=request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), save_state=False, ) @@ -417,7 +434,7 @@ async def update_project_node_state( async def update_project_node_progress( app: web.Application, user_id: int, project_id: str, node_id: str, progress: float -) -> Optional[dict]: +) -> dict | None: log.debug( "updating node %s progress in project %s for user %s with %s", node_id, @@ -445,8 +462,8 @@ async def update_project_node_outputs( user_id: int, project_id: str, node_id: str, - new_outputs: Optional[dict], - new_run_hash: Optional[str], + new_outputs: dict | None, + new_run_hash: str | None, ) -> tuple[dict, list[str]]: """ Updates outputs of a given node in a project with 'data' @@ -599,7 +616,7 @@ async def try_open_project_for_user( project_uuid: str, client_session_id: str, app: web.Application, - max_number_of_studies_per_user: Optional[int], + max_number_of_studies_per_user: int | None, ) -> bool: try: async with lock_with_notification( @@ -674,6 +691,7 @@ async def try_close_project_for_user( project_uuid: str, client_session_id: str, app: web.Application, + simcore_user_agent: str, ): with managed_resource(user_id, client_session_id, app) as rt: user_to_session_ids: list[UserSessionID] = await rt.find_users_of_resource( @@ -699,7 +717,9 @@ async def try_close_project_for_user( if not user_to_session_ids: # NOTE: depending on the garbage collector speed, it might already be removing it fire_and_forget_task( - remove_project_dynamic_services(user_id, project_uuid, app), + remove_project_dynamic_services( + user_id, project_uuid, app, simcore_user_agent + ), task_suffix_name=f"remove_project_dynamic_services_{user_id=}_{project_uuid=}", fire_and_forget_tasks_collection=app[APP_FIRE_AND_FORGET_TASKS_KEY], ) @@ -733,7 +753,7 @@ async def _get_project_lock_state( f"{project_uuid=}", f"{user_id=}", ) - prj_locked_state: Optional[ProjectLocked] = await get_project_locked_state( + prj_locked_state: ProjectLocked | None = await get_project_locked_state( app, project_uuid ) if prj_locked_state: @@ -978,8 +998,9 @@ async def remove_project_dynamic_services( user_id: int, project_uuid: str, app: web.Application, + simcore_user_agent: str, notify_users: bool = True, - user_name: Optional[UserNameDict] = None, + user_name: UserNameDict | None = None, ) -> None: """ @@ -997,7 +1018,7 @@ async def remove_project_dynamic_services( user_name_data: UserNameDict = user_name or await get_user_name(app, user_id) - user_role: Optional[UserRole] = None + user_role: UserRole | None = None try: user_role = await get_user_role(app, user_id) except UserNotFoundError: @@ -1025,6 +1046,7 @@ async def remove_project_dynamic_services( app=app, user_id=user_id, project_id=project_uuid, + simcore_user_agent=simcore_user_agent, save_state=save_state, ) @@ -1037,7 +1059,7 @@ async def remove_project_dynamic_services( async def notify_project_state_update( app: web.Application, project: dict, - notify_only_user: Optional[int] = None, + notify_only_user: int | None = None, ) -> None: messages: list[SocketMessageDict] = [ { @@ -1065,7 +1087,7 @@ async def notify_project_node_update( app: web.Application, project: dict, node_id: str, - errors: Optional[list[ErrorDict]], + errors: list[ErrorDict] | None, ) -> None: rooms_to_notify = [ f"{gid}" for gid, rights in project["accessRights"].items() if rights["read"] diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py b/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py index aa2afa1871e..d73bccf5fae 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_handlers.py @@ -12,6 +12,10 @@ from models_library.projects_state import ProjectState from servicelib.aiohttp.requests_validation import parse_request_path_parameters_as from servicelib.aiohttp.web_exceptions_extension import HTTPLocked +from servicelib.common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_SIMCORE_USER_AGENT, +) from servicelib.json_serialization import json_dumps from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from simcore_postgres_database.models.users import UserRole @@ -99,7 +103,7 @@ async def open_project(request: web.Request) -> web.Response: # and let's update the project last change timestamp await projects_api.update_project_last_change_timestamp( - request.app, f"{path_params.project_id}" + request.app, path_params.project_id ) # notify users that project is now opened @@ -125,6 +129,9 @@ async def open_project(request: web.Request) -> web.Response: project_uuid=f"{path_params.project_id}", client_session_id=client_session_id, app=request.app, + simcore_user_agent=request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), ) raise web.HTTPServiceUnavailable( reason="Unexpected error while starting services." @@ -146,7 +153,6 @@ async def open_project(request: web.Request) -> web.Response: @login_required @permission_required("project.close") async def close_project(request: web.Request) -> web.Response: - req_ctx = RequestContext.parse_obj(request) path_params = parse_request_path_parameters_as(ProjectPathParams, request) @@ -169,6 +175,9 @@ async def close_project(request: web.Request) -> web.Response: f"{path_params.project_id}", client_session_id, request.app, + simcore_user_agent=request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), ) raise web.HTTPNoContent(content_type=MIMETYPE_APPLICATION_JSON) except ProjectNotFoundError as exc: diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_handlers_crud.py b/services/web/server/src/simcore_service_webserver/projects/projects_handlers_crud.py index cf831e0515a..cacb7b0c1e1 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_handlers_crud.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_handlers_crud.py @@ -7,12 +7,12 @@ import json import logging from contextlib import AsyncExitStack -from typing import Any, Coroutine, Optional +from typing import Any, Coroutine from aiohttp import web from jsonschema import ValidationError as JsonSchemaValidationError from models_library.projects import ProjectID -from models_library.projects_state import ProjectStatus +from models_library.projects_state import ProjectLocked, ProjectStatus from models_library.rest_pagination import DEFAULT_NUMBER_OF_ITEMS_PER_PAGE, Page from models_library.rest_pagination_utils import paginate_data from models_library.users import UserID @@ -26,6 +26,10 @@ parse_request_path_parameters_as, parse_request_query_parameters_as, ) +from servicelib.common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_SIMCORE_USER_AGENT, +) from servicelib.json_serialization import json_dumps from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from servicelib.rest_constants import RESPONSE_MODEL_POLICY @@ -52,7 +56,6 @@ from .projects_exceptions import ( ProjectDeleteError, ProjectInvalidRightsError, - ProjectLockError, ProjectNotFoundError, ) from .projects_nodes_utils import update_frontend_outputs @@ -105,7 +108,7 @@ class Config: class _ProjectCreateParams(BaseModel): - from_study: Optional[ProjectID] = Field( + from_study: ProjectID | None = Field( None, description="Option to create a project from existing template or study: from_study={study_uuid}", ) @@ -147,6 +150,9 @@ async def create_projects(request: web.Request): query_params=query_params, predefined_project=predefined_project, product_name=req_ctx.product_name, + simcore_user_agent=request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), fire_and_forget=True, ) @@ -159,7 +165,7 @@ async def _prepare_project_copy( as_template: bool, deep_copy: bool, task_progress: TaskProgress, -) -> tuple[ProjectDict, Optional[Coroutine[Any, Any, None]]]: +) -> tuple[ProjectDict, Coroutine[Any, Any, None] | None]: source_project = await projects_api.get_project_for_user( app, project_uuid=f"{src_project_uuid}", @@ -249,8 +255,9 @@ async def _create_projects( app: web.Application, query_params: _ProjectCreateParams, request_context: RequestContext, - predefined_project: Optional[ProjectDict], + predefined_project: ProjectDict | None, product_name: str, + simcore_user_agent: str, ) -> None: """ @@ -345,7 +352,7 @@ async def _create_projects( # FIXME: If cancelled during shutdown, cancellation of all_tasks will produce "new tasks"! if prj_uuid := new_project.get("uuid"): await projects_api.submit_delete_project_task( - app, prj_uuid, request_context.user_id + app, prj_uuid, request_context.user_id, simcore_user_agent ) raise @@ -713,7 +720,7 @@ async def delete_project(request: web.Request): "It cannot be deleted until the project is closed." ) - project_locked_state: Optional[ProjectLockError] + project_locked_state: ProjectLocked | None if project_locked_state := await get_project_locked_state( app=request.app, project_uuid=path_params.project_id ): @@ -722,7 +729,12 @@ async def delete_project(request: web.Request): ) await projects_api.submit_delete_project_task( - request.app, path_params.project_id, req_ctx.user_id + request.app, + path_params.project_id, + req_ctx.user_id, + request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), ) except ProjectInvalidRightsError as err: diff --git a/services/web/server/src/simcore_service_webserver/projects/projects_nodes_handlers.py b/services/web/server/src/simcore_service_webserver/projects/projects_nodes_handlers.py index 2992d50e6aa..dfd8ebdb56f 100644 --- a/services/web/server/src/simcore_service_webserver/projects/projects_nodes_handlers.py +++ b/services/web/server/src/simcore_service_webserver/projects/projects_nodes_handlers.py @@ -4,7 +4,6 @@ import json import logging -from typing import Optional, Union from aiohttp import web from models_library.projects_nodes import NodeID @@ -19,6 +18,10 @@ parse_request_body_as, parse_request_path_parameters_as, ) +from servicelib.common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_SIMCORE_USER_AGENT, +) from servicelib.json_serialization import json_dumps from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON from simcore_postgres_database.models.users import UserRole @@ -51,7 +54,7 @@ class _CreateNodeBody(BaseModel): service_key: ServiceKey service_version: ServiceVersion - service_id: Optional[str] = None + service_id: str | None = None @routes.post(f"/{VTAG}/projects/{{project_id}}/nodes") @@ -131,7 +134,7 @@ async def get_node(request: web.Request) -> web.Response: ) # NOTE: for legacy services a redirect to director-v0 is made - service_data: Union[dict, list] = await director_v2_api.get_dynamic_service( + service_data: dict | list = await director_v2_api.get_dynamic_service( app=request.app, node_uuid=f"{path_params.node_id}" ) @@ -286,6 +289,9 @@ async def stop_node(request: web.Request) -> web.Response: path_params=path_params, app=request.app, service_uuid=f"{path_params.node_id}", + simcore_user_agent=request.headers.get( + X_SIMCORE_USER_AGENT, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ), save_state=save_state, fire_and_forget=True, ) diff --git a/services/web/server/tests/integration/02/test_rabbit.py b/services/web/server/tests/integration/02/test_rabbit.py index a7bfd27cb07..c9043302a1a 100644 --- a/services/web/server/tests/integration/02/test_rabbit.py +++ b/services/web/server/tests/integration/02/test_rabbit.py @@ -30,6 +30,7 @@ from pytest_simcore.helpers.utils_login import UserInfoDict from redis import Redis from servicelib.aiohttp.application import create_safe_application +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from settings_library.rabbit import RabbitSettings from simcore_postgres_database.models.comp_tasks import NodeClass from simcore_service_webserver.application_settings import setup_settings @@ -134,6 +135,7 @@ async def _publish_in_rabbit( service_type=NodeClass.COMPUTATIONAL.value, service_key="some/service/awesome/key", service_tag="some-awesome-tag", + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, ) instrumentation_stop_message.metrics = "service_stopped" instrumentation_stop_message.result = RunningState.SUCCESS diff --git a/services/web/server/tests/unit/isolated/test_director_v2_core_dynamic_services.py b/services/web/server/tests/unit/isolated/test_director_v2_core_dynamic_services.py index 0b8fd8087e4..5fd07b70374 100644 --- a/services/web/server/tests/unit/isolated/test_director_v2_core_dynamic_services.py +++ b/services/web/server/tests/unit/isolated/test_director_v2_core_dynamic_services.py @@ -7,8 +7,13 @@ import pytest from aiohttp.web import Application, HTTPConflict, HTTPNoContent from faker import Faker +from models_library.projects_nodes_io import NodeIDStr from pytest_mock.plugin import MockerFixture from servicelib.aiohttp.application import create_safe_application +from servicelib.common_headers import ( + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + X_SIMCORE_USER_AGENT, +) from simcore_service_webserver import director_v2_core_dynamic_services from simcore_service_webserver.application_settings import setup_settings from simcore_service_webserver.director_v2_exceptions import ( @@ -42,7 +47,10 @@ async def test_stop_dynamic_service_signature( can_save: bool, ): await director_v2_core_dynamic_services.stop_dynamic_service( - app, node_uuid, save_state=can_save + app, + NodeIDStr(node_uuid), + UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + save_state=can_save, ) mocked_director_v2_request.assert_called_with( app, @@ -51,6 +59,7 @@ async def test_stop_dynamic_service_signature( f"http://director-v2:8000/v2/dynamic_services/{node_uuid}?can_save={f'{can_save}'.lower()}" ), expected_status=HTTPNoContent, + headers={X_SIMCORE_USER_AGENT: UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE}, timeout=3610, on_error={ HTTPConflict.status_code: ( diff --git a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_studies_access.py b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_studies_access.py index d8186c008ac..a9f68d8aad5 100644 --- a/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_studies_access.py +++ b/services/web/server/tests/unit/with_dbs/01/studies_dispatcher/test_studies_dispatcher_studies_access.py @@ -27,12 +27,13 @@ from servicelib.aiohttp.long_running_tasks.client import LRTask from servicelib.aiohttp.long_running_tasks.server import TaskProgress from servicelib.aiohttp.rest_responses import unwrap_envelope +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_service_webserver.projects.project_models import ProjectDict from simcore_service_webserver.projects.projects_api import submit_delete_project_task from simcore_service_webserver.users_api import delete_user, get_user_role -async def _get_user_projects(client): +async def _get_user_projects(client) -> list[ProjectDict]: url = client.app.router["list_projects"].url_for() resp = await client.get(url.with_query(type="user")) @@ -71,7 +72,6 @@ async def published_project( tests_data_dir: Path, osparc_product_name: str, ) -> AsyncIterator[ProjectDict]: - project_data = deepcopy(fake_project) project_data["name"] = "Published project" project_data["uuid"] = "e2e38eee-c569-4e55-b104-70d159e49c87" @@ -189,7 +189,6 @@ def _assert_redirected_to_error_page( async def _assert_redirected_to_study( response: ClientResponse, session: ClientSession ) -> str: - # https://docs.aiohttp.org/en/stable/client_advanced.html#redirection-history assert len(response.history) == 1, "Is a re-direction" @@ -260,18 +259,18 @@ async def test_access_study_anonymously( catalog_subsystem_mock([published_project]) assert not _is_user_authenticated(client.session), "Is anonymous" - + assert client.app study_url = client.app.router["get_redirection_to_study_page"].url_for( id=published_project["uuid"] ) - resp = await client.get(study_url) + resp = await client.get(f"{study_url}") expected_prj_id = await _assert_redirected_to_study(resp, client.session) # has auto logged in as guest? me_url = client.app.router["get_my_profile"].url_for() - resp = await client.get(me_url) + resp = await client.get(f"{me_url}") data, _ = await assert_status(resp, web.HTTPOk) assert data["login"].endswith("guest-at-osparc.io") @@ -291,6 +290,7 @@ async def test_access_study_anonymously( @pytest.fixture async def auto_delete_projects(client: TestClient) -> AsyncIterator[None]: + assert client.app yield await delete_all_projects(client.app) @@ -308,13 +308,14 @@ async def test_access_study_by_logged_user( # needed to cleanup the locks between parametrizations redis_locks_client: AsyncIterator[aioredis.Redis], ): + assert client.app catalog_subsystem_mock([published_project]) assert _is_user_authenticated(client.session), "Is already logged-in" study_url = client.app.router["get_redirection_to_study_page"].url_for( id=published_project["uuid"] ) - resp = await client.get(study_url) + resp = await client.get(f"{study_url}") await _assert_redirected_to_study(resp, client.session) # user has a copy of the template project @@ -341,18 +342,19 @@ async def test_access_cookie_of_expired_user( ): catalog_subsystem_mock([published_project]) # emulates issue #1570 + assert client.app # nosec app: web.Application = client.app study_url = app.router["get_redirection_to_study_page"].url_for( id=published_project["uuid"] ) - resp = await client.get(study_url) + resp = await client.get(f"{study_url}") await _assert_redirected_to_study(resp, client.session) # Expects valid cookie and GUEST access me_url = app.router["get_my_profile"].url_for() - resp = await client.get(me_url) + resp = await client.get(f"{me_url}") data, _ = await assert_status(resp, web.HTTPOk) assert await get_user_role(app, data["id"]) == UserRole.GUEST @@ -369,7 +371,9 @@ async def enforce_garbage_collect_guest(uid): prj_id = projects[0]["uuid"] - delete_task = await submit_delete_project_task(app, prj_id, uid) + delete_task = await submit_delete_project_task( + app, prj_id, uid, UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE + ) await delete_task await delete_user(app, uid) @@ -379,15 +383,15 @@ async def enforce_garbage_collect_guest(uid): user_email = data["login"] # Now this should be non -authorized - resp = await client.get(me_url) + resp = await client.get(f"{me_url}") await assert_status(resp, web.HTTPUnauthorized) # But still can access as a new user - resp = await client.get(study_url) + resp = await client.get(f"{study_url}") await _assert_redirected_to_study(resp, client.session) # as a guest user - resp = await client.get(me_url) + resp = await client.get(f"{me_url}") data, _ = await assert_status(resp, web.HTTPOk) assert await get_user_role(app, data["id"]) == UserRole.GUEST diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py b/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py index e4b055b9059..0e4ec78bdc4 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__delete.py @@ -20,6 +20,7 @@ MockedStorageSubsystem, standard_role_response, ) +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.products import products from simcore_postgres_database.models.projects_to_products import projects_to_products from simcore_service_webserver._meta import api_version_prefix @@ -84,6 +85,7 @@ async def test_delete_project( call( app=client.app, service_uuid=service["service_uuid"], + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=True, progress=mock_progress_bar.sub_progress(1), ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__open_close.py b/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__open_close.py index 9cdabb6eb09..bce78057bde 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__open_close.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_handlers__open_close.py @@ -9,7 +9,7 @@ import time from copy import deepcopy from datetime import datetime, timedelta -from typing import Any, Awaitable, Callable, Iterator, Optional, Union +from typing import Any, Awaitable, Callable, Iterator from unittest import mock from unittest.mock import call @@ -40,6 +40,7 @@ standard_role_response, ) from servicelib.aiohttp.web_exceptions_extension import HTTPLocked +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.products import products from simcore_service_webserver.db_models import UserRole from simcore_service_webserver.projects.project_models import ProjectDict @@ -79,7 +80,7 @@ def _extract(dikt, keys): async def _list_projects( client, expected: type[web.HTTPException], - query_parameters: Optional[dict] = None, + query_parameters: dict | None = None, ) -> list[dict[str, Any]]: # GET /v0/projects url = client.app.router["list_projects"].url_for() @@ -112,8 +113,8 @@ async def _connect_websocket( check_connection: bool, client, client_id: str, - events: Optional[dict[str, Callable]] = None, -) -> Optional[socketio.AsyncClient]: + events: dict[str, Callable] | None = None, +) -> socketio.AsyncClient | None: try: sio = await socketio_client_factory(client_id, client) assert sio.sid @@ -130,7 +131,7 @@ async def _open_project( client, client_id: str, project: dict, - expected: Union[type[web.HTTPException], list[type[web.HTTPException]]], + expected: type[web.HTTPException] | list[type[web.HTTPException]], ) -> tuple[dict, dict]: url = client.app.router["open_project"].url_for(project_id=project["uuid"]) resp = await client.post(url, json=client_id) @@ -342,7 +343,7 @@ async def test_open_project( service_version=service["version"], user_id=logged_user["id"], request_scheme=request_scheme, - request_simcore_user_agent="", + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, request_dns=request_dns, product_name=osparc_product_name, service_resources=ServiceResourcesDictHelpers.create_jsonable( @@ -407,7 +408,7 @@ async def test_open_template_project_for_edition( service_version=service["version"], user_id=logged_user["id"], request_scheme=request_scheme, - request_simcore_user_agent="", + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, request_dns=request_dns, service_resources=ServiceResourcesDictHelpers.create_jsonable( mock_service_resources @@ -685,6 +686,7 @@ async def test_close_project( call( app=client.server.app, service_uuid=service["service_uuid"], + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=True, progress=mock_progress_bar.sub_progress(1), ) diff --git a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py index 9430e3ed921..2c78790ec30 100644 --- a/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py +++ b/services/web/server/tests/unit/with_dbs/02/test_projects_nodes_handler.py @@ -24,6 +24,7 @@ MockedStorageSubsystem, standard_role_response, ) +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_postgres_database.models.projects import projects as projects_db_model from simcore_service_webserver.db_models import UserRole from simcore_service_webserver.projects.project_models import ProjectDict @@ -502,7 +503,12 @@ async def test_delete_node( if node_id in running_dy_services: mocked_director_v2_api[ "director_v2_api.stop_dynamic_service" - ].assert_called_once_with(mock.ANY, node_id, save_state=False) + ].assert_called_once_with( + mock.ANY, + node_id, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, + save_state=False, + ) mocked_director_v2_api["director_v2_api.stop_dynamic_service"].reset_mock() else: mocked_director_v2_api[ diff --git a/services/web/server/tests/unit/with_dbs/03/garbage_collector/test_resource_manager.py b/services/web/server/tests/unit/with_dbs/03/garbage_collector/test_resource_manager.py index 6b50dc9ca8b..2d8825cabd5 100644 --- a/services/web/server/tests/unit/with_dbs/03/garbage_collector/test_resource_manager.py +++ b/services/web/server/tests/unit/with_dbs/03/garbage_collector/test_resource_manager.py @@ -28,6 +28,7 @@ from redis.asyncio import Redis from servicelib.aiohttp.application import create_safe_application from servicelib.aiohttp.application_setup import is_setup_completed +from servicelib.common_headers import UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE from simcore_service_webserver import garbage_collector_core from simcore_service_webserver._meta import API_VTAG from simcore_service_webserver.application_settings import setup_settings @@ -107,7 +108,6 @@ def client( mock_rabbitmq: None, mock_progress_bar: Any, ) -> TestClient: - cfg = deepcopy(app_cfg) assert cfg["rest"]["version"] == API_VTAG assert cfg["rest"]["enabled"] @@ -220,7 +220,6 @@ async def test_anonymous_websocket_connection( security_cookie_factory: Callable, mocker, ): - sio = socketio.AsyncClient( ssl_verify=False ) # enginio 3.10.0 introduced ssl verification @@ -500,6 +499,7 @@ async def test_interactive_services_removed_after_logout( ].assert_awaited_with( app=client.app, service_uuid=service["service_uuid"], + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=expected_save_state, progress=mock_progress_bar.sub_progress(1), ) @@ -527,7 +527,6 @@ async def test_interactive_services_remain_after_websocket_reconnection_from_2_t open_project: Callable, mock_progress_bar: Any, ): - # login - logged_user fixture # create empty study - empty_user_project fixture # create dynamic service - create_dynamic_service_mock fixture @@ -611,6 +610,7 @@ async def test_interactive_services_remain_after_websocket_reconnection_from_2_t calls = [ call( app=client.server.app, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=expected_save_state, service_uuid=service["service_uuid"], progress=mock_progress_bar.sub_progress(1), @@ -704,6 +704,7 @@ async def test_interactive_services_removed_per_project( call( app=client.server.app, service_uuid=service1["service_uuid"], + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=expected_save_state, progress=mock_progress_bar.sub_progress(1), ) @@ -730,12 +731,14 @@ async def test_interactive_services_removed_per_project( call( app=client.server.app, service_uuid=service2["service_uuid"], + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=expected_save_state, progress=mock_progress_bar.sub_progress(1), ), call( app=client.server.app, service_uuid=service3["service_uuid"], + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=expected_save_state, progress=mock_progress_bar.sub_progress(1), ), @@ -852,6 +855,7 @@ async def test_websocket_disconnected_remove_or_maintain_files_based_on_role( calls = [ call( app=client.server.app, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, save_state=expected_save_state, service_uuid=service["service_uuid"], progress=mock_progress_bar.sub_progress(1), @@ -891,6 +895,7 @@ async def test_regression_removing_unexisting_user( app=client.app, project_uuid=empty_user_project["uuid"], user_id=logged_user["id"], + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, ) await delete_task # remove user @@ -901,6 +906,7 @@ async def test_regression_removing_unexisting_user( user_id=logged_user["id"], project_uuid=empty_user_project["uuid"], app=client.app, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, ) with pytest.raises(ProjectNotFoundError): await remove_project_dynamic_services( @@ -908,6 +914,7 @@ async def test_regression_removing_unexisting_user( project_uuid=empty_user_project["uuid"], app=client.app, user_name={"first_name": "my name is", "last_name": "pytest"}, + simcore_user_agent=UNDEFINED_DEFAULT_SIMCORE_USER_AGENT_VALUE, ) # since the call to delete is happening as fire and forget task, let's wait until it is done async for attempt in AsyncRetrying(**_TENACITY_ASSERT_RETRY):