✨Add simcore_user_agent and product_name to user service labels

packages/models-library/src/models_library/docker.py

@@ -1,11 +1,13 @@
 import re
-from typing import Optional
+import warnings
+from typing import Any, Optional
 
 from models_library.generated_models.docker_rest_api import Task
+from models_library.products import ProductName
 from models_library.projects import ProjectID
 from models_library.projects_nodes import NodeID
 from models_library.users import UserID
-from pydantic import BaseModel, ConstrainedStr, Field
+from pydantic import BaseModel, ConstrainedStr, Field, root_validator
 
 from .basic_regex import DOCKER_GENERIC_TAG_KEY_RE, DOCKER_LABEL_KEY_REGEX
 
@@ -30,10 +32,32 @@ class SimcoreServiceDockerLabelKeys(BaseModel):
     project_id: ProjectID = Field(..., alias="study_id")
     node_id: NodeID = Field(..., alias="uuid")
 
+    product_name: ProductName
+    simcore_user_agent: str
+
+    @root_validator(pre=True)
+    @classmethod
+    def ensure_defaults(cls, values: dict[str, Any]) -> dict[str, Any]:
+        warnings.warn(
+            (
+                "Once https://github.com/ITISFoundation/osparc-simcore/pull/3990 "
+                "reaches production this entire root_validator function "
+                "can be safely removed. Please check "
+                "https://github.com/ITISFoundation/osparc-simcore/issues/3996"
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        if values.get("product_name", None) is None:
+            values["product_name"] = "opsarc"
+        if values.get("simcore_user_agent", None) is None:
+            values["simcore_user_agent"] = ""
+        return values
+
     def to_docker_labels(self) -> dict[str, str]:
         """returns a dictionary of strings as required by docker"""
         std_export = self.dict(by_alias=True)
-        return {k: f"{v}" for k, v in std_export.items()}
+        return {k: f"{v}" for k, v in sorted(std_export.items())}
 
     @classmethod
     def from_docker_task(cls, docker_task: Task) -> "SimcoreServiceDockerLabelKeys":

packages/models-library/tests/test_docker.py

@@ -3,6 +3,8 @@
 # pylint: disable=unused-variable
 
 
+from typing import Any
+
 import pytest
 from faker import Faker
 from models_library.docker import (
@@ -12,6 +14,8 @@
 )
 from pydantic import ValidationError, parse_obj_as
 
+_faker = Faker()
+
 
 @pytest.mark.parametrize(
     "label_key, valid",
@@ -101,18 +105,33 @@ def test_docker_generic_tag(image_name: str, valid: bool):
             parse_obj_as(DockerGenericTag, image_name)
 
 
-@pytest.fixture
-def osparc_docker_label_keys(
-    faker: Faker,
-) -> SimcoreServiceDockerLabelKeys:
-    return SimcoreServiceDockerLabelKeys.parse_obj(
-        dict(user_id=faker.pyint(), project_id=faker.uuid4(), node_id=faker.uuid4())
+@pytest.mark.parametrize(
+    "obj_data",
+    [
+        pytest.param(
+            {
+                "user_id": _faker.pyint(),
+                "project_id": _faker.uuid4(),
+                "node_id": _faker.uuid4(),
+            },
+            id="parse_existing_service_labels",
+        ),
+        pytest.param(
+            {
+                "user_id": _faker.pyint(),
+                "project_id": _faker.uuid4(),
+                "node_id": _faker.uuid4(),
+                "product": "test_p",
+                "simcore_user_agent": "a-test-puppet",
+            },
+            id="parse_new_service_labels",
+        ),
+    ],
+)
+def test_simcore_service_docker_label_keys(obj_data: dict[str, Any]):
+    simcore_service_docker_label_keys = SimcoreServiceDockerLabelKeys.parse_obj(
+        obj_data
     )
-
-
-def test_osparc_docker_label_keys_to_docker_labels(
-    osparc_docker_label_keys: SimcoreServiceDockerLabelKeys,
-):
-    exported_dict = osparc_docker_label_keys.to_docker_labels()
+    exported_dict = simcore_service_docker_label_keys.to_docker_labels()
     assert all(isinstance(v, str) for v in exported_dict.values())
     assert parse_obj_as(SimcoreServiceDockerLabelKeys, exported_dict)

packages/service-library/src/servicelib/aiohttp/monitoring.py

@@ -21,6 +21,7 @@
 from prometheus_client.registry import CollectorRegistry
 from servicelib.aiohttp.typing_extension import Handler
 
+from ..common_headers import X_SIMCORE_USER_AGENT
 from ..logging_utils import log_catch
 
 log = logging.getLogger(__name__)
@@ -112,7 +113,6 @@
 kPLATFORM_COLLECTOR = f"{__name__}.collector_platform"
 kGC_COLLECTOR = f"{__name__}.collector_gc"
 
-SIMCORE_USER_AGENT_HEADER: Final[str] = "X-Simcore-User-Agent"
 UNDEFINED_REGULAR_USER_AGENT: Final[str] = "undefined"
 
 
@@ -168,16 +168,12 @@ async def middleware_handler(request: web.Request, handler: Handler):
                 app_name,
                 request.method,
                 canonical_endpoint,
-                request.headers.get(
-                    SIMCORE_USER_AGENT_HEADER, UNDEFINED_REGULAR_USER_AGENT
-                ),
+                request.headers.get(X_SIMCORE_USER_AGENT, UNDEFINED_REGULAR_USER_AGENT),
             ).track_inprogress(), response_summary.labels(
                 app_name,
                 request.method,
                 canonical_endpoint,
-                request.headers.get(
-                    SIMCORE_USER_AGENT_HEADER, UNDEFINED_REGULAR_USER_AGENT
-                ),
+                request.headers.get(X_SIMCORE_USER_AGENT, UNDEFINED_REGULAR_USER_AGENT),
             ).time():
                 resp = await handler(request)
 
@@ -213,9 +209,7 @@ async def middleware_handler(request: web.Request, handler: Handler):
                 request.method,
                 canonical_endpoint,
                 resp.status,
-                request.headers.get(
-                    SIMCORE_USER_AGENT_HEADER, UNDEFINED_REGULAR_USER_AGENT
-                ),
+                request.headers.get(X_SIMCORE_USER_AGENT, UNDEFINED_REGULAR_USER_AGENT),
             ).inc()
 
             if exit_middleware_cb:

packages/service-library/src/servicelib/common_headers.py

@@ -0,0 +1,6 @@
+from typing import Final
+
+X_DYNAMIC_SIDECAR_REQUEST_DNS: Final[str] = "X-Dynamic-Sidecar-Request-DNS"
+X_DYNAMIC_SIDECAR_REQUEST_SCHEME: Final[str] = "X-Dynamic-Sidecar-Request-Scheme"
+X_FORWARDED_PROTO: Final[str] = "X-Forwarded-Proto"
+X_SIMCORE_USER_AGENT: Final[str] = "X-Simcore-User-Agent"

packages/service-library/tests/aiohttp/test_monitoring.py

@@ -11,11 +11,8 @@
 from aiohttp.test_utils import TestClient
 from faker import Faker
 from prometheus_client.parser import text_string_to_metric_families
-from servicelib.aiohttp.monitoring import (
-    SIMCORE_USER_AGENT_HEADER,
-    UNDEFINED_REGULAR_USER_AGENT,
-    setup_monitoring,
-)
+from servicelib.aiohttp.monitoring import UNDEFINED_REGULAR_USER_AGENT, setup_monitoring
+from servicelib.common_headers import X_SIMCORE_USER_AGENT
 
 
 @pytest.fixture
@@ -120,7 +117,7 @@ async def test_request_with_simcore_user_agent(client: TestClient, faker: Faker)
     faker_simcore_user_agent = faker.name()
     response = await client.get(
         "/monitored_request",
-        headers={SIMCORE_USER_AGENT_HEADER: faker_simcore_user_agent},
+        headers={X_SIMCORE_USER_AGENT: faker_simcore_user_agent},
     )
     assert response.status == web.HTTPOk.status_code
 

services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py

@@ -101,13 +101,13 @@ async def create_dynamic_service(
     service: DynamicServiceCreate,
     x_dynamic_sidecar_request_dns: str = Header(...),
     x_dynamic_sidecar_request_scheme: str = Header(...),
+    x_simcore_user_agent: str = Header(...),
     director_v0_client: DirectorV0Client = Depends(get_director_v0_client),
     dynamic_services_settings: DynamicServicesSettings = Depends(
         get_dynamic_services_settings
     ),
     scheduler: DynamicSidecarsScheduler = Depends(get_scheduler),
 ) -> Union[DynamicServiceGet, RedirectResponse]:
-
     simcore_service_labels: SimcoreServiceLabels = (
         await director_v0_client.get_service_labels(
             service=ServiceKeyVersion(key=service.key, version=service.version)
@@ -141,6 +141,7 @@ async def create_dynamic_service(
             port=dynamic_services_settings.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_PORT,
             request_dns=x_dynamic_sidecar_request_dns,
             request_scheme=x_dynamic_sidecar_request_scheme,
+            request_simcore_user_agent=x_simcore_user_agent,
         )
 
     return cast(DynamicServiceGet, await scheduler.get_stack_status(service.node_uuid))
@@ -156,7 +157,6 @@ async def get_dynamic_sidecar_status(
     director_v0_client: DirectorV0Client = Depends(get_director_v0_client),
     scheduler: DynamicSidecarsScheduler = Depends(get_scheduler),
 ) -> Union[DynamicServiceGet, RedirectResponse]:
-
     try:
         return cast(DynamicServiceGet, await scheduler.get_stack_status(node_uuid))
     except DynamicSidecarNotFoundError:
@@ -317,7 +317,6 @@ async def update_projects_networks(
     director_v0_client: DirectorV0Client = Depends(get_director_v0_client),
     rabbitmq_client: RabbitMQClient = Depends(get_rabbitmq_client),
 ) -> None:
-
     await projects_networks.update_from_workbench(
         projects_networks_repository=projects_networks_repository,
         projects_repository=projects_repository,

services/director-v2/src/simcore_service_director_v2/models/schemas/dynamic_services/scheduler.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import warnings
 from enum import Enum
 from functools import cached_property
 from typing import Any, Mapping, Optional
@@ -16,7 +17,15 @@
 )
 from models_library.services import RunID
 from models_library.services_resources import ServiceResourcesDict
-from pydantic import AnyHttpUrl, BaseModel, Extra, Field, constr, parse_obj_as
+from pydantic import (
+    AnyHttpUrl,
+    BaseModel,
+    Extra,
+    Field,
+    constr,
+    parse_obj_as,
+    root_validator,
+)
 from servicelib.error_codes import ErrorCodeStr
 from servicelib.exception_utils import DelayedExceptionHandler
 
@@ -370,14 +379,38 @@ def endpoint(self) -> AnyHttpUrl:
     request_scheme: str = Field(
         ..., description="used when configuring the CORS options on the proxy"
     )
+    request_simcore_user_agent: str = Field(
+        ...,
+        description="used as label to filter out the metrics from the cAdvisor prometheus metrics",
+    )
     proxy_service_name: str = Field(None, description="service name given to the proxy")
 
-    product_name: Optional[str] = Field(
+    product_name: str = Field(
         None,
         description="Current product upon which this service is scheduled. "
         "If set to None, the current product is undefined. Mostly for backwards compatibility",
     )
 
+    @root_validator(pre=True)
+    @classmethod
+    def _ensure_legacy_format_compatibility(cls, values):
+        warnings.warn(
+            (
+                "Once https://github.com/ITISFoundation/osparc-simcore/pull/3990 "
+                "reaches production this entire root_validator function "
+                "can be safely removed. Please check "
+                "https://github.com/ITISFoundation/osparc-simcore/issues/3996"
+            ),
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        request_simcore_user_agent: Optional[str] = values.get(
+            "request_simcore_user_agent"
+        )
+        if not request_simcore_user_agent:
+            values["request_simcore_user_agent"] = ""
+        return values
+
     @classmethod
     def from_http_request(
         # pylint: disable=too-many-arguments
@@ -387,10 +420,10 @@ def from_http_request(
         port: PortInt,
         request_dns: str,
         request_scheme: str,
+        request_simcore_user_agent: str,
         run_id: Optional[UUID] = None,
     ) -> "SchedulerData":
         # This constructor method sets current product
-        assert service.product_name is not None  # nosec
         names_helper = DynamicSidecarNamesHelper.make(service.node_uuid)
 
         obj_dict = dict(
@@ -413,6 +446,7 @@ def from_http_request(
             request_dns=request_dns,
             request_scheme=request_scheme,
             proxy_service_name=names_helper.proxy_service_name,
+            request_simcore_user_agent=request_simcore_user_agent,
             dynamic_sidecar={},
         )
         if run_id:

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_compose_specs.py

@@ -4,6 +4,7 @@
 
 from fastapi.applications import FastAPI
 from models_library.docker import SimcoreServiceDockerLabelKeys
+from models_library.products import ProductName
 from models_library.projects import ProjectID
 from models_library.projects_nodes_io import NodeID
 from models_library.service_settings_labels import (
@@ -192,12 +193,18 @@ def _update_container_labels(
     user_id: UserID,
     project_id: ProjectID,
     node_id: NodeID,
+    simcore_user_agent: str,
+    product_name: ProductName,
 ) -> None:
     for spec in service_spec["services"].values():
         labels: list[str] = spec.setdefault("labels", [])
 
         label_keys = SimcoreServiceDockerLabelKeys(
-            user_id=user_id, study_id=project_id, uuid=node_id
+            user_id=user_id,
+            study_id=project_id,
+            uuid=node_id,
+            simcore_user_agent=simcore_user_agent,
+            product_name=product_name,
         )
         docker_labels = [f"{k}={v}" for k, v in label_keys.to_docker_labels().items()]
 
@@ -219,10 +226,11 @@ def assemble_spec(
     service_resources: ServiceResourcesDict,
     simcore_service_labels: SimcoreServiceLabels,
     allow_internet_access: bool,
-    product_name: str,
+    product_name: ProductName,
     user_id: UserID,
     project_id: ProjectID,
     node_id: NodeID,
+    simcore_user_agent: str,
 ) -> str:
     """
     returns a docker-compose spec used by
@@ -286,6 +294,8 @@ def assemble_spec(
         user_id=user_id,
         project_id=project_id,
         node_id=node_id,
+        product_name=product_name,
+        simcore_user_agent=simcore_user_agent,
     )
 
     # TODO: will be used in next PR

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_abc.py

@@ -79,6 +79,7 @@ async def add_service(
         port: PortInt,
         request_dns: str,
         request_scheme: str,
+        request_simcore_user_agent: str,
     ) -> None:
         """
         Adds a new service.

services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py

@@ -106,7 +106,6 @@ async def will_trigger(cls, app: FastAPI, scheduler_data: SchedulerData) -> bool
 
     @classmethod
     async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
-
         # instrumentation
         message = InstrumentationRabbitMessage(
             metrics="service_started",
@@ -482,6 +481,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None:
             user_id=scheduler_data.user_id,
             project_id=scheduler_data.project_id,
             node_id=scheduler_data.node_uuid,
+            simcore_user_agent=scheduler_data.request_simcore_user_agent,
         )
 
         logger.debug(