From 192005e66d888226bef99f0280f46e46f910356a Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 11 Oct 2024 14:35:29 +0200 Subject: [PATCH 1/8] notify frontend about current efs disk space --- .../core/application.py | 3 + .../services/notifier_setup.py | 59 +++++++++++++++++++ .../services/process_messages.py | 17 +++++- 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py index 6bf2833ed02..612e6d44fc7 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py @@ -17,6 +17,7 @@ from ..services.efs_manager_setup import setup as setup_efs_manager from ..services.modules.rabbitmq import setup as setup_rabbitmq from ..services.modules.redis import setup as setup_redis +from ..services.notifier_setup import setup_notifier from ..services.process_messages_setup import setup as setup_process_messages from .settings import ApplicationSettings @@ -52,6 +53,8 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_background_tasks(app) setup_process_messages(app) + setup_notifier(app) + # EVENTS async def _on_startup() -> None: print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py new file mode 100644 index 00000000000..a96ec1abc83 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py @@ -0,0 +1,59 @@ +import contextlib +from typing import Final + +import socketio # type: ignore[import-untyped] +from fastapi import FastAPI +from fastapi.encoders import jsonable_encoder +from models_library.api_schemas_webserver.socketio import SocketIORoomStr +from models_library.projects_nodes_io import NodeID +from models_library.users import UserID +from pydantic import BaseModel, ByteSize, Field +from servicelib.fastapi.app_state import SingletonInAppStateMixin + +SOCKET_IO_EFS_DISK_USAGE_EVENT: Final[str] = "efsNodeDiskUsage" + + +class EfsNodeDiskUsage(BaseModel): + node_id: NodeID + used: ByteSize = Field(description="used space") + free: ByteSize = Field(description="remaining space") + total: ByteSize = Field(description="total space = free + used") + used_percent: float = Field( + gte=0.00, + lte=100.00, + description="Percent of used space relative to the total space", + ) + + +class Notifier(SingletonInAppStateMixin): + app_state_name: str = "notifier" + + def __init__(self, sio_manager: socketio.AsyncAioPikaManager): + self._sio_manager = sio_manager + + async def notify_service_efs_disk_usage( + self, user_id: UserID, efs_node_disk_usage: EfsNodeDiskUsage + ) -> None: + await self._sio_manager.emit( + SOCKET_IO_EFS_DISK_USAGE_EVENT, + data=jsonable_encoder(efs_node_disk_usage), + room=SocketIORoomStr.from_user_id(user_id), + ) + + +def setup_notifier(app: FastAPI): + async def _on_startup() -> None: + assert app.state.external_socketio # nosec + + notifier = Notifier( + sio_manager=app.state.external_socketio, + ) + notifier.set_to_app_state(app) + assert Notifier.get_from_app_state(app) == notifier # nosec + + async def _on_shutdown() -> None: + with contextlib.suppress(AttributeError): + Notifier.pop_from_app_state(app) + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py index 11c7781bbae..329dea64044 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py @@ -2,9 +2,13 @@ from fastapi import FastAPI from models_library.rabbitmq_messages import DynamicServiceRunningMessage -from pydantic import parse_raw_as +from pydantic import ByteSize, parse_raw_as from servicelib.logging_utils import log_context from simcore_service_efs_guardian.services.modules.redis import get_redis_lock_client +from simcore_service_efs_guardian.services.notifier_setup import ( + EfsNodeDiskUsage, + Notifier, +) from ..core.settings import get_application_settings from ..services.efs_manager import EfsManager @@ -49,6 +53,17 @@ async def process_dynamic_service_running_message(app: FastAPI, data: bytes) -> rabbit_message.node_id, rabbit_message.user_id, ) + efs_node_disk_usage = EfsNodeDiskUsage( + node_id=rabbit_message.node_id, + used=size, + free=ByteSize(settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES - size), + total=settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES, + used_percent=round(size / settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES, 2), + ) + notifier: Notifier = Notifier.get_from_app_state(app) + await notifier.notify_service_efs_disk_usage( + user_id=rabbit_message.user_id, efs_node_disk_usage=efs_node_disk_usage + ) if size > settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES: msg = f"Removing write permissions inside of EFS starts for project ID: {rabbit_message.project_id}, node ID: {rabbit_message.node_id}, current user: {rabbit_message.user_id}, size: {size}, upper limit: {settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES}" From 1a1e51c81dcbc972a253275efd3536ba17cb8819 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 11 Oct 2024 16:38:47 +0200 Subject: [PATCH 2/8] addinh unit test --- services/efs-guardian/requirements/_base.in | 1 + services/efs-guardian/requirements/_base.txt | 151 ++++++++++-------- services/efs-guardian/requirements/_test.txt | 76 +++++---- services/efs-guardian/requirements/_tools.txt | 30 ++-- .../core/application.py | 4 +- .../services/notifications/__init__.py | 3 + .../services/notifications/_notifier.py | 59 +++++++ .../services/notifications/_setup.py | 15 ++ .../services/notifications/_socketio.py | 32 ++++ .../services/process_messages.py | 3 +- services/efs-guardian/tests/unit/conftest.py | 2 + .../tests/unit/test_process_messages.py | 116 ++++++++++++++ 12 files changed, 371 insertions(+), 121 deletions(-) create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py create mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py create mode 100644 services/efs-guardian/tests/unit/test_process_messages.py diff --git a/services/efs-guardian/requirements/_base.in b/services/efs-guardian/requirements/_base.in index 84e8460fa05..a1b2f8e7f1b 100644 --- a/services/efs-guardian/requirements/_base.in +++ b/services/efs-guardian/requirements/_base.in @@ -16,3 +16,4 @@ fastapi packaging +python-socketio diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index 9f98cc7a99d..49799985267 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -1,12 +1,12 @@ -aio-pika==9.4.1 +aio-pika==9.4.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aioboto3==13.1.0 +aioboto3==13.1.1 # via -r requirements/../../../packages/aws-library/requirements/_base.in aiobotocore==2.13.1 # via aioboto3 -aiocache==0.12.2 +aiocache==0.12.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in @@ -15,16 +15,18 @@ aiodebug==2.3.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.21.0 +aiodocker==0.23.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiofiles==23.2.1 +aiofiles==24.1.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aioboto3 -aiohttp==3.9.5 +aiohappyeyeballs==2.4.3 + # via aiohttp +aiohttp==3.10.10 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -40,13 +42,13 @@ aiohttp==3.9.5 # -c requirements/../../../requirements/constraints.txt # aiobotocore # aiodocker -aioitertools==0.11.0 +aioitertools==0.12.0 # via aiobotocore -aiormq==6.8.0 +aiormq==6.8.1 # via aio-pika aiosignal==1.3.1 # via aiohttp -anyio==4.4.0 +anyio==4.6.0 # via # fast-depends # faststream @@ -63,21 +65,23 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi -attrs==23.2.0 +attrs==24.2.0 # via # aiohttp # jsonschema # referencing -boto3==1.34.106 +bidict==0.23.1 + # via python-socketio +boto3==1.34.131 # via aiobotocore -botocore==1.34.106 +botocore==1.34.131 # via # aiobotocore # boto3 # s3transfer -botocore-stubs==1.34.94 +botocore-stubs==1.35.38 # via types-aiobotocore -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -94,7 +98,7 @@ certifi==2024.2.2 # httpcore # httpx # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -106,11 +110,11 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -dnspython==2.6.1 +dnspython==2.7.0 # via email-validator -email-validator==2.1.1 +email-validator==2.2.0 # via pydantic -fast-depends==2.4.3 +fast-depends==2.4.11 # via faststream fastapi==0.99.1 # via @@ -129,7 +133,7 @@ fastapi==0.99.1 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.9 +faststream==0.5.25 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -141,15 +145,16 @@ googleapis-common-protos==1.65.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -grpcio==1.66.0 +grpcio==1.66.2 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.5 + # wsproto +httpcore==1.0.6 # via httpx -httpx==0.27.0 +httpx==0.27.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -164,20 +169,20 @@ httpx==0.27.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in -idna==3.7 +idna==3.10 # via # anyio # email-validator # httpx # requests # yarl -importlib-metadata==8.0.0 +importlib-metadata==8.4.0 # via opentelemetry-api jmespath==1.0.1 # via # boto3 # botocore -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -189,11 +194,11 @@ markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl -opentelemetry-api==1.26.0 +opentelemetry-api==1.27.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -205,54 +210,54 @@ opentelemetry-api==1.26.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.26.0 +opentelemetry-exporter-otlp==1.27.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.26.0 +opentelemetry-exporter-otlp-proto-common==1.27.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.26.0 +opentelemetry-exporter-otlp-proto-grpc==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.26.0 +opentelemetry-exporter-otlp-proto-http==1.27.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.47b0 +opentelemetry-instrumentation-asgi==0.48b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.47b0 +opentelemetry-instrumentation-fastapi==0.48b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.47b0 +opentelemetry-instrumentation-requests==0.48b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.26.0 +opentelemetry-proto==1.27.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.26.0 +opentelemetry-sdk==1.27.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.47b0 +opentelemetry-semantic-conventions==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.47b0 +opentelemetry-util-http==0.48b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -orjson==3.10.3 +orjson==3.10.7 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -270,17 +275,19 @@ orjson==3.10.3 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -packaging==24.0 +packaging==24.1 # via -r requirements/_base.in pamqp==3.3.0 # via aiormq -prometheus-client==0.20.0 +prometheus-client==0.21.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -protobuf==4.25.4 +propcache==0.2.0 + # via yarl +protobuf==4.25.5 # via # googleapis-common-protos # opentelemetry-proto @@ -288,7 +295,7 @@ psutil==6.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -pydantic==1.10.15 +pydantic==1.10.18 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -318,7 +325,7 @@ pydantic==1.10.15 # fastapi pygments==2.18.0 # via rich -pyinstrument==4.6.2 +pyinstrument==4.7.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -326,7 +333,11 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore -pyyaml==6.0.1 +python-engineio==4.9.1 + # via python-socketio +python-socketio==5.11.4 + # via -r requirements/_base.in +pyyaml==6.0.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -342,7 +353,7 @@ pyyaml==6.0.1 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -redis==5.0.4 +redis==5.1.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -370,25 +381,27 @@ repro-zipfile==0.3.1 # -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.7.1 +rich==13.9.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.18.1 +rpds-py==0.20.0 # via # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.3 # via boto3 -setuptools==74.0.0 +setuptools==75.1.0 # via opentelemetry-instrumentation -sh==2.0.6 +sh==2.1.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer +simple-websocket==1.1.0 + # via python-engineio six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -410,41 +423,39 @@ starlette==0.27.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi -tenacity==8.5.0 +tenacity==9.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -toolz==0.12.1 +toolz==1.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -tqdm==4.66.4 +tqdm==4.66.5 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.3 +typer==0.12.5 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in - # faststream -types-aiobotocore==2.13.0 +types-aiobotocore==2.15.2 # via -r requirements/../../../packages/aws-library/requirements/_base.in -types-aiobotocore-ec2==2.13.0 +types-aiobotocore-ec2==2.15.2 # via types-aiobotocore -types-aiobotocore-s3==2.13.0 +types-aiobotocore-s3==2.15.2 # via types-aiobotocore -types-aiobotocore-ssm==2.13.1 +types-aiobotocore-ssm==2.15.2 # via types-aiobotocore -types-awscrt==0.20.9 +types-awscrt==0.22.0 # via botocore-stubs -types-python-dateutil==2.9.0.20240316 +types-python-dateutil==2.9.0.20241003 # via arrow -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via # aiodebug - # aiodocker # fastapi # faststream # opentelemetry-sdk @@ -454,7 +465,7 @@ typing-extensions==4.11.0 # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -470,17 +481,19 @@ urllib3==2.2.1 # -c requirements/../../../requirements/constraints.txt # botocore # requests -uvicorn==0.30.0 +uvicorn==0.31.1 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in wrapt==1.16.0 # via # aiobotocore # deprecated # opentelemetry-instrumentation -yarl==1.9.4 +wsproto==1.2.0 + # via simple-websocket +yarl==1.14.0 # via # aio-pika # aiohttp # aiormq -zipp==3.20.1 +zipp==3.20.2 # via importlib-metadata diff --git a/services/efs-guardian/requirements/_test.txt b/services/efs-guardian/requirements/_test.txt index efd05c557f6..3c576c58c46 100644 --- a/services/efs-guardian/requirements/_test.txt +++ b/services/efs-guardian/requirements/_test.txt @@ -1,8 +1,12 @@ -aiodocker==0.21.0 +aiodocker==0.23.0 # via # -c requirements/_base.txt # -r requirements/_test.in -aiohttp==3.9.5 +aiohappyeyeballs==2.4.3 + # via + # -c requirements/_base.txt + # aiohttp +aiohttp==3.10.10 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -13,37 +17,37 @@ aiosignal==1.3.1 # aiohttp antlr4-python3-runtime==4.13.2 # via moto -anyio==4.4.0 +anyio==4.6.0 # via # -c requirements/_base.txt # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -attrs==23.2.0 +attrs==24.2.0 # via # -c requirements/_base.txt # aiohttp # jsonschema # referencing -aws-sam-translator==1.89.0 +aws-sam-translator==1.91.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto blinker==1.8.2 # via flask -boto3==1.34.106 +boto3==1.34.131 # via # -c requirements/_base.txt # aws-sam-translator # moto -botocore==1.34.106 +botocore==1.34.131 # via # -c requirements/_base.txt # aws-xray-sdk # boto3 # moto # s3transfer -certifi==2024.2.2 +certifi==2024.8.30 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -52,9 +56,9 @@ certifi==2024.2.2 # requests cffi==1.17.1 # via cryptography -cfn-lint==1.10.3 +cfn-lint==1.16.1 # via moto -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -c requirements/_base.txt # requests @@ -62,7 +66,7 @@ click==8.1.7 # via # -c requirements/_base.txt # flask -coverage==7.6.1 +coverage==7.6.2 # via # -r requirements/_test.in # pytest-cov @@ -71,7 +75,7 @@ cryptography==43.0.1 # -c requirements/../../../requirements/constraints.txt # joserfc # moto -debugpy==1.8.5 +debugpy==1.8.7 # via -r requirements/_test.in deepdiff==8.0.1 # via -r requirements/_test.in @@ -79,9 +83,9 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -faker==29.0.0 +faker==30.3.0 # via -r requirements/_test.in -fakeredis==2.24.1 +fakeredis==2.25.1 # via -r requirements/_test.in flask==3.0.3 # via @@ -100,17 +104,17 @@ h11==0.14.0 # via # -c requirements/_base.txt # httpcore -httpcore==1.0.5 +httpcore==1.0.6 # via # -c requirements/_base.txt # httpx -httpx==0.27.0 +httpx==0.27.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in # respx -idna==3.7 +idna==3.10 # via # -c requirements/_base.txt # anyio @@ -141,7 +145,7 @@ jsonpath-ng==1.6.1 # via moto jsonpointer==3.0.0 # via jsonpatch -jsonschema==4.22.0 +jsonschema==4.23.0 # via # -c requirements/_base.txt # aws-sam-translator @@ -158,20 +162,20 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 # via fakeredis -markupsafe==2.1.5 +markupsafe==3.0.1 # via # jinja2 # werkzeug -moto==5.0.15 +moto==5.0.16 # via -r requirements/_test.in mpmath==1.3.0 # via sympy -multidict==6.0.5 +multidict==6.1.0 # via # -c requirements/_base.txt # aiohttp # yarl -networkx==3.3 +networkx==3.4 # via cfn-lint openapi-schema-validator==0.6.2 # via openapi-spec-validator @@ -179,7 +183,7 @@ openapi-spec-validator==0.7.1 # via moto orderly-set==5.2.2 # via deepdiff -packaging==24.0 +packaging==24.1 # via # -c requirements/_base.txt # pytest @@ -191,6 +195,10 @@ pluggy==1.5.0 # via pytest ply==3.11 # via jsonpath-ng +propcache==0.2.0 + # via + # -c requirements/_base.txt + # yarl psutil==6.0.0 # via # -c requirements/_base.txt @@ -199,7 +207,7 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==1.10.15 +pydantic==1.10.18 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -230,7 +238,7 @@ python-dateutil==2.9.0.post0 # moto python-dotenv==1.0.1 # via -r requirements/_test.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -239,7 +247,7 @@ pyyaml==6.0.1 # jsonschema-path # moto # responses -redis==5.0.4 +redis==5.1.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -265,16 +273,16 @@ respx==0.21.1 # via -r requirements/_test.in rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.18.1 +rpds-py==0.20.0 # via # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.1 +s3transfer==0.10.3 # via # -c requirements/_base.txt # boto3 -setuptools==74.0.0 +setuptools==75.1.0 # via # -c requirements/_base.txt # moto @@ -293,14 +301,14 @@ sortedcontainers==2.4.0 # via fakeredis sympy==1.13.3 # via cfn-lint -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via # -c requirements/_base.txt - # aiodocker # aws-sam-translator # cfn-lint + # faker # pydantic -urllib3==2.2.1 +urllib3==2.2.3 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -316,9 +324,9 @@ wrapt==1.16.0 # via # -c requirements/_base.txt # aws-xray-sdk -xmltodict==0.13.0 +xmltodict==0.14.1 # via moto -yarl==1.9.4 +yarl==1.14.0 # via # -c requirements/_base.txt # aiohttp diff --git a/services/efs-guardian/requirements/_tools.txt b/services/efs-guardian/requirements/_tools.txt index 97a49efc2eb..2fbc38251f3 100644 --- a/services/efs-guardian/requirements/_tools.txt +++ b/services/efs-guardian/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.4 +astroid==3.3.5 # via pylint -black==24.8.0 +black==24.10.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2 +build==1.2.2.post1 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -14,9 +14,9 @@ click==8.1.7 # -c requirements/_test.txt # black # pip-tools -dill==0.3.8 +dill==0.3.9 # via pylint -distlib==0.3.8 +distlib==0.3.9 # via virtualenv filelock==3.16.1 # via virtualenv @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.0 +packaging==24.1 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -53,38 +53,38 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==3.8.0 +pre-commit==4.0.1 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.0 +pylint==3.3.1 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.1.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit # watchdog -ruff==0.6.7 +ruff==0.6.9 # via -r requirements/../../../requirements/devenv.txt -setuptools==74.0.0 +setuptools==75.1.0 # via # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 # via pylint -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.5 +virtualenv==20.26.6 # via pre-commit -watchdog==5.0.2 +watchdog==5.0.3 # via -r requirements/_tools.in wheel==0.44.0 # via pip-tools diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py index 612e6d44fc7..46c14496ee9 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py @@ -17,7 +17,7 @@ from ..services.efs_manager_setup import setup as setup_efs_manager from ..services.modules.rabbitmq import setup as setup_rabbitmq from ..services.modules.redis import setup as setup_redis -from ..services.notifier_setup import setup_notifier +from ..services.notifications._setup import setup_notifications from ..services.process_messages_setup import setup as setup_process_messages from .settings import ApplicationSettings @@ -53,7 +53,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_background_tasks(app) setup_process_messages(app) - setup_notifier(app) + setup_notifications(app) # EVENTS async def _on_startup() -> None: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py new file mode 100644 index 00000000000..d270853f5a7 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py @@ -0,0 +1,3 @@ +from ._setup import setup_notifications + +__all__: tuple[str, ...] = ("setup_notifications",) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py new file mode 100644 index 00000000000..a96ec1abc83 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py @@ -0,0 +1,59 @@ +import contextlib +from typing import Final + +import socketio # type: ignore[import-untyped] +from fastapi import FastAPI +from fastapi.encoders import jsonable_encoder +from models_library.api_schemas_webserver.socketio import SocketIORoomStr +from models_library.projects_nodes_io import NodeID +from models_library.users import UserID +from pydantic import BaseModel, ByteSize, Field +from servicelib.fastapi.app_state import SingletonInAppStateMixin + +SOCKET_IO_EFS_DISK_USAGE_EVENT: Final[str] = "efsNodeDiskUsage" + + +class EfsNodeDiskUsage(BaseModel): + node_id: NodeID + used: ByteSize = Field(description="used space") + free: ByteSize = Field(description="remaining space") + total: ByteSize = Field(description="total space = free + used") + used_percent: float = Field( + gte=0.00, + lte=100.00, + description="Percent of used space relative to the total space", + ) + + +class Notifier(SingletonInAppStateMixin): + app_state_name: str = "notifier" + + def __init__(self, sio_manager: socketio.AsyncAioPikaManager): + self._sio_manager = sio_manager + + async def notify_service_efs_disk_usage( + self, user_id: UserID, efs_node_disk_usage: EfsNodeDiskUsage + ) -> None: + await self._sio_manager.emit( + SOCKET_IO_EFS_DISK_USAGE_EVENT, + data=jsonable_encoder(efs_node_disk_usage), + room=SocketIORoomStr.from_user_id(user_id), + ) + + +def setup_notifier(app: FastAPI): + async def _on_startup() -> None: + assert app.state.external_socketio # nosec + + notifier = Notifier( + sio_manager=app.state.external_socketio, + ) + notifier.set_to_app_state(app) + assert Notifier.get_from_app_state(app) == notifier # nosec + + async def _on_shutdown() -> None: + with contextlib.suppress(AttributeError): + Notifier.pop_from_app_state(app) + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py new file mode 100644 index 00000000000..6de0fae307f --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py @@ -0,0 +1,15 @@ +import logging + +from fastapi import FastAPI +from servicelib.logging_utils import log_context + +from ..notifications._notifier import setup_notifier +from ..notifications._socketio import setup_socketio + +_logger = logging.getLogger(__name__) + + +def setup_notifications(app: FastAPI) -> None: + with log_context(_logger, logging.INFO, "setup notifications"): + setup_socketio(app) + setup_notifier(app) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py new file mode 100644 index 00000000000..5ceb1049148 --- /dev/null +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py @@ -0,0 +1,32 @@ +import logging + +import socketio # type: ignore[import-untyped] +from fastapi import FastAPI +from servicelib.socketio_utils import cleanup_socketio_async_pubsub_manager + +from ...core.settings import ApplicationSettings + +_logger = logging.getLogger(__name__) + + +def setup_socketio(app: FastAPI): + settings: ApplicationSettings = app.state.settings + + async def _on_startup() -> None: + assert app.state.rabbitmq_client # nosec + + # Connect to the as an external process in write-only mode + # SEE https://python-socketio.readthedocs.io/en/stable/server.html#emitting-from-external-processes + assert settings.EFS_GUARDIAN_RABBITMQ # nosec + app.state.external_socketio = socketio.AsyncAioPikaManager( + url=settings.EFS_GUARDIAN_RABBITMQ.dsn, logger=_logger, write_only=True + ) + + async def _on_shutdown() -> None: + if external_socketio := getattr(app.state, "external_socketio"): # noqa: B009 + await cleanup_socketio_async_pubsub_manager( + server_manager=external_socketio + ) + + app.add_event_handler("startup", _on_startup) + app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py index 329dea64044..fc601b4dbd9 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py @@ -53,12 +53,13 @@ async def process_dynamic_service_running_message(app: FastAPI, data: bytes) -> rabbit_message.node_id, rabbit_message.user_id, ) + percentage = round(size / settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES * 100, 2) efs_node_disk_usage = EfsNodeDiskUsage( node_id=rabbit_message.node_id, used=size, free=ByteSize(settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES - size), total=settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES, - used_percent=round(size / settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES, 2), + used_percent=min(percentage, 100.0), ) notifier: Notifier = Notifier.get_from_app_state(app) await notifier.notify_service_efs_disk_usage( diff --git a/services/efs-guardian/tests/unit/conftest.py b/services/efs-guardian/tests/unit/conftest.py index da4196ea859..05842003790 100644 --- a/services/efs-guardian/tests/unit/conftest.py +++ b/services/efs-guardian/tests/unit/conftest.py @@ -33,6 +33,8 @@ "pytest_simcore.docker_swarm", "pytest_simcore.environment_configs", "pytest_simcore.faker_projects_data", + "pytest_simcore.faker_users_data", + "pytest_simcore.faker_products_data", "pytest_simcore.pydantic_models", "pytest_simcore.pytest_global_environs", "pytest_simcore.rabbit_service", diff --git a/services/efs-guardian/tests/unit/test_process_messages.py b/services/efs-guardian/tests/unit/test_process_messages.py new file mode 100644 index 00000000000..facaf9380aa --- /dev/null +++ b/services/efs-guardian/tests/unit/test_process_messages.py @@ -0,0 +1,116 @@ +# pylint: disable=protected-access +# pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments +# pylint: disable=unused-argument +# pylint: disable=unused-variable + + +from unittest.mock import AsyncMock, patch + +import pytest +from faker import Faker +from fastapi import FastAPI +from models_library.products import ProductName +from models_library.rabbitmq_messages import DynamicServiceRunningMessage +from models_library.users import UserID +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_efs_guardian.services.efs_manager import NodeID, ProjectID +from simcore_service_efs_guardian.services.process_messages import ( + process_dynamic_service_running_message, +) + +pytest_simcore_core_services_selection = ["rabbit"] +pytest_simcore_ops_services_selection = [] + + +@pytest.fixture +def app_environment( + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, + rabbit_env_vars_dict: EnvVarsDict, +) -> EnvVarsDict: + return setenvs_from_dict( + monkeypatch, + { + **app_environment, + **rabbit_env_vars_dict, + "EFS_DEFAULT_USER_SERVICE_SIZE_BYTES": "10000", + }, + ) + + +@patch( + "simcore_service_efs_guardian.services.process_messages.Notifier.get_from_app_state" +) +async def test_process_msg( + mock_notifier_get_from_app_state, + faker: Faker, + mocked_redis_server: None, + app: FastAPI, + cleanup: None, + project_id: ProjectID, + node_id: NodeID, + user_id: UserID, + product_name: ProductName, +): + # Create mock data for the message + model_instance = DynamicServiceRunningMessage( + project_id=project_id, + node_id=node_id, + user_id=user_id, + product_name=product_name, + ) + json_str = model_instance.json() + model_bytes = json_str.encode("utf-8") + + # Mock efs_manager and its methods + mock_efs_manager = AsyncMock() + app.state.efs_manager = mock_efs_manager + mock_efs_manager.check_project_node_data_directory_exits.return_value = True + mock_efs_manager.get_project_node_data_size.return_value = 4000 + + # Mock the Notifier + mock_notifier = AsyncMock() + mock_notifier_get_from_app_state.return_value = mock_notifier + + result = await process_dynamic_service_running_message(app, data=model_bytes) + + # Check the actual arguments passed to notify_service_efs_disk_usage + _, kwargs = mock_notifier.notify_service_efs_disk_usage.call_args + assert kwargs["user_id"] == user_id + efs_node_disk_usage = kwargs["efs_node_disk_usage"] + + # Check EfsNodeDiskUsage values + assert efs_node_disk_usage.node_id == node_id + assert efs_node_disk_usage.used == 4000 + assert efs_node_disk_usage.free == 6000 + assert efs_node_disk_usage.total == 10000 + assert efs_node_disk_usage.used_percent == 40.0 + + # Check that the function returns True + assert result is True + + +@pytest.mark.skip() +async def test_process_msg__dir_not_exists( + mocked_redis_server: None, + app: FastAPI, + cleanup: None, + project_id: ProjectID, + node_id: NodeID, + user_id: UserID, + product_name: ProductName, +): + # Create mock data for the message + model_instance = DynamicServiceRunningMessage( + project_id=project_id, + node_id=node_id, + user_id=user_id, + product_name=product_name, + ) + json_str = model_instance.json() + model_bytes = json_str.encode("utf-8") + + result = await process_dynamic_service_running_message(app, data=model_bytes) + assert result is True From 508d024bce380be2d9bb6c2b146106ba4119b01c Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 11 Oct 2024 17:41:56 +0200 Subject: [PATCH 3/8] fix function --- .../simcore_service_efs_guardian/services/process_messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py index fc601b4dbd9..2d49fe52c45 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py @@ -57,7 +57,7 @@ async def process_dynamic_service_running_message(app: FastAPI, data: bytes) -> efs_node_disk_usage = EfsNodeDiskUsage( node_id=rabbit_message.node_id, used=size, - free=ByteSize(settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES - size), + free=ByteSize(max(settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES - size, 0)), total=settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES, used_percent=min(percentage, 100.0), ) From 55462010454f33f54e7996c33722012ff67beca6 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Fri, 11 Oct 2024 17:44:12 +0200 Subject: [PATCH 4/8] remove redundant code --- .../services/notifier_setup.py | 59 ------------------- 1 file changed, 59 deletions(-) delete mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py deleted file mode 100644 index a96ec1abc83..00000000000 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifier_setup.py +++ /dev/null @@ -1,59 +0,0 @@ -import contextlib -from typing import Final - -import socketio # type: ignore[import-untyped] -from fastapi import FastAPI -from fastapi.encoders import jsonable_encoder -from models_library.api_schemas_webserver.socketio import SocketIORoomStr -from models_library.projects_nodes_io import NodeID -from models_library.users import UserID -from pydantic import BaseModel, ByteSize, Field -from servicelib.fastapi.app_state import SingletonInAppStateMixin - -SOCKET_IO_EFS_DISK_USAGE_EVENT: Final[str] = "efsNodeDiskUsage" - - -class EfsNodeDiskUsage(BaseModel): - node_id: NodeID - used: ByteSize = Field(description="used space") - free: ByteSize = Field(description="remaining space") - total: ByteSize = Field(description="total space = free + used") - used_percent: float = Field( - gte=0.00, - lte=100.00, - description="Percent of used space relative to the total space", - ) - - -class Notifier(SingletonInAppStateMixin): - app_state_name: str = "notifier" - - def __init__(self, sio_manager: socketio.AsyncAioPikaManager): - self._sio_manager = sio_manager - - async def notify_service_efs_disk_usage( - self, user_id: UserID, efs_node_disk_usage: EfsNodeDiskUsage - ) -> None: - await self._sio_manager.emit( - SOCKET_IO_EFS_DISK_USAGE_EVENT, - data=jsonable_encoder(efs_node_disk_usage), - room=SocketIORoomStr.from_user_id(user_id), - ) - - -def setup_notifier(app: FastAPI): - async def _on_startup() -> None: - assert app.state.external_socketio # nosec - - notifier = Notifier( - sio_manager=app.state.external_socketio, - ) - notifier.set_to_app_state(app) - assert Notifier.get_from_app_state(app) == notifier # nosec - - async def _on_shutdown() -> None: - with contextlib.suppress(AttributeError): - Notifier.pop_from_app_state(app) - - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) From 9c72d06f54a545ad8e519784fe12619907a728f6 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 22 Oct 2024 13:12:45 +0200 Subject: [PATCH 5/8] update to new style --- services/efs-guardian/requirements/_base.in | 1 - services/efs-guardian/requirements/_base.txt | 151 ++++++++---------- services/efs-guardian/requirements/_test.txt | 76 ++++----- services/efs-guardian/requirements/_tools.txt | 32 ++-- .../core/application.py | 3 - .../services/efs_manager.py | 28 ++++ .../services/modules/rabbitmq.py | 10 ++ .../services/notifications/__init__.py | 3 - .../services/notifications/_notifier.py | 59 ------- .../services/notifications/_setup.py | 15 -- .../services/notifications/_socketio.py | 32 ---- .../services/process_messages.py | 39 +++-- .../tests/unit/test_process_messages.py | 35 ++-- 13 files changed, 195 insertions(+), 289 deletions(-) delete mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py delete mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py delete mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py delete mode 100644 services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py diff --git a/services/efs-guardian/requirements/_base.in b/services/efs-guardian/requirements/_base.in index a1b2f8e7f1b..84e8460fa05 100644 --- a/services/efs-guardian/requirements/_base.in +++ b/services/efs-guardian/requirements/_base.in @@ -16,4 +16,3 @@ fastapi packaging -python-socketio diff --git a/services/efs-guardian/requirements/_base.txt b/services/efs-guardian/requirements/_base.txt index 49799985267..9f98cc7a99d 100644 --- a/services/efs-guardian/requirements/_base.txt +++ b/services/efs-guardian/requirements/_base.txt @@ -1,12 +1,12 @@ -aio-pika==9.4.3 +aio-pika==9.4.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aioboto3==13.1.1 +aioboto3==13.1.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in aiobotocore==2.13.1 # via aioboto3 -aiocache==0.12.3 +aiocache==0.12.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/_base.in @@ -15,18 +15,16 @@ aiodebug==2.3.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiodocker==0.23.0 +aiodocker==0.21.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -aiofiles==24.1.0 +aiofiles==23.2.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # aioboto3 -aiohappyeyeballs==2.4.3 - # via aiohttp -aiohttp==3.10.10 +aiohttp==3.9.5 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -42,13 +40,13 @@ aiohttp==3.10.10 # -c requirements/../../../requirements/constraints.txt # aiobotocore # aiodocker -aioitertools==0.12.0 +aioitertools==0.11.0 # via aiobotocore -aiormq==6.8.1 +aiormq==6.8.0 # via aio-pika aiosignal==1.3.1 # via aiohttp -anyio==4.6.0 +anyio==4.4.0 # via # fast-depends # faststream @@ -65,23 +63,21 @@ arrow==1.3.0 # -r requirements/../../../packages/service-library/requirements/_base.in asgiref==3.8.1 # via opentelemetry-instrumentation-asgi -attrs==24.2.0 +attrs==23.2.0 # via # aiohttp # jsonschema # referencing -bidict==0.23.1 - # via python-socketio -boto3==1.34.131 +boto3==1.34.106 # via aiobotocore -botocore==1.34.131 +botocore==1.34.106 # via # aiobotocore # boto3 # s3transfer -botocore-stubs==1.35.38 +botocore-stubs==1.34.94 # via types-aiobotocore -certifi==2024.8.30 +certifi==2024.2.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -98,7 +94,7 @@ certifi==2024.8.30 # httpcore # httpx # requests -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -110,11 +106,11 @@ deprecated==1.2.14 # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http # opentelemetry-semantic-conventions -dnspython==2.7.0 +dnspython==2.6.1 # via email-validator -email-validator==2.2.0 +email-validator==2.1.1 # via pydantic -fast-depends==2.4.11 +fast-depends==2.4.3 # via faststream fastapi==0.99.1 # via @@ -133,7 +129,7 @@ fastapi==0.99.1 # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator -faststream==0.5.25 +faststream==0.5.9 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -145,16 +141,15 @@ googleapis-common-protos==1.65.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -grpcio==1.66.2 +grpcio==1.66.0 # via opentelemetry-exporter-otlp-proto-grpc h11==0.14.0 # via # httpcore # uvicorn - # wsproto -httpcore==1.0.6 +httpcore==1.0.5 # via httpx -httpx==0.27.2 +httpx==0.27.0 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -169,20 +164,20 @@ httpx==0.27.2 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in -idna==3.10 +idna==3.7 # via # anyio # email-validator # httpx # requests # yarl -importlib-metadata==8.4.0 +importlib-metadata==8.0.0 # via opentelemetry-api jmespath==1.0.1 # via # boto3 # botocore -jsonschema==4.23.0 +jsonschema==4.22.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in @@ -194,11 +189,11 @@ markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -multidict==6.1.0 +multidict==6.0.5 # via # aiohttp # yarl -opentelemetry-api==1.27.0 +opentelemetry-api==1.26.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -210,54 +205,54 @@ opentelemetry-api==1.27.0 # opentelemetry-instrumentation-requests # opentelemetry-sdk # opentelemetry-semantic-conventions -opentelemetry-exporter-otlp==1.27.0 +opentelemetry-exporter-otlp==1.26.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-exporter-otlp-proto-common==1.27.0 +opentelemetry-exporter-otlp-proto-common==1.26.0 # via # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-exporter-otlp-proto-grpc==1.27.0 +opentelemetry-exporter-otlp-proto-grpc==1.26.0 # via opentelemetry-exporter-otlp -opentelemetry-exporter-otlp-proto-http==1.27.0 +opentelemetry-exporter-otlp-proto-http==1.26.0 # via opentelemetry-exporter-otlp -opentelemetry-instrumentation==0.48b0 +opentelemetry-instrumentation==0.47b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -opentelemetry-instrumentation-asgi==0.48b0 +opentelemetry-instrumentation-asgi==0.47b0 # via opentelemetry-instrumentation-fastapi -opentelemetry-instrumentation-fastapi==0.48b0 +opentelemetry-instrumentation-fastapi==0.47b0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -opentelemetry-instrumentation-requests==0.48b0 +opentelemetry-instrumentation-requests==0.47b0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -opentelemetry-proto==1.27.0 +opentelemetry-proto==1.26.0 # via # opentelemetry-exporter-otlp-proto-common # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-sdk==1.27.0 +opentelemetry-sdk==1.26.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -opentelemetry-semantic-conventions==0.48b0 +opentelemetry-semantic-conventions==0.47b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests # opentelemetry-sdk -opentelemetry-util-http==0.48b0 +opentelemetry-util-http==0.47b0 # via # opentelemetry-instrumentation-asgi # opentelemetry-instrumentation-fastapi # opentelemetry-instrumentation-requests -orjson==3.10.7 +orjson==3.10.3 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -275,19 +270,17 @@ orjson==3.10.7 # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in -packaging==24.1 +packaging==24.0 # via -r requirements/_base.in pamqp==3.3.0 # via aiormq -prometheus-client==0.21.0 +prometheus-client==0.20.0 # via # -r requirements/../../../packages/service-library/requirements/_fastapi.in # prometheus-fastapi-instrumentator prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in -propcache==0.2.0 - # via yarl -protobuf==4.25.5 +protobuf==4.25.4 # via # googleapis-common-protos # opentelemetry-proto @@ -295,7 +288,7 @@ psutil==6.0.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -pydantic==1.10.18 +pydantic==1.10.15 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -325,7 +318,7 @@ pydantic==1.10.18 # fastapi pygments==2.18.0 # via rich -pyinstrument==4.7.3 +pyinstrument==4.6.2 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in @@ -333,11 +326,7 @@ python-dateutil==2.9.0.post0 # via # arrow # botocore -python-engineio==4.9.1 - # via python-socketio -python-socketio==5.11.4 - # via -r requirements/_base.in -pyyaml==6.0.2 +pyyaml==6.0.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -353,7 +342,7 @@ pyyaml==6.0.2 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -redis==5.1.1 +redis==5.0.4 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -381,27 +370,25 @@ repro-zipfile==0.3.1 # -r requirements/../../../packages/service-library/requirements/_base.in requests==2.32.3 # via opentelemetry-exporter-otlp-proto-http -rich==13.9.2 +rich==13.7.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in # typer -rpds-py==0.20.0 +rpds-py==0.18.1 # via # jsonschema # referencing -s3transfer==0.10.3 +s3transfer==0.10.1 # via boto3 -setuptools==75.1.0 +setuptools==74.0.0 # via opentelemetry-instrumentation -sh==2.1.0 +sh==2.0.6 # via -r requirements/../../../packages/aws-library/requirements/_base.in shellingham==1.5.4 # via typer -simple-websocket==1.1.0 - # via python-engineio six==1.16.0 # via python-dateutil sniffio==1.3.1 @@ -423,39 +410,41 @@ starlette==0.27.0 # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt # fastapi -tenacity==9.0.0 +tenacity==8.5.0 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -toolz==1.0.0 +toolz==0.12.1 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -tqdm==4.66.5 +tqdm==4.66.4 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in -typer==0.12.5 +typer==0.12.3 # via # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -r requirements/../../../packages/settings-library/requirements/_base.in -types-aiobotocore==2.15.2 + # faststream +types-aiobotocore==2.13.0 # via -r requirements/../../../packages/aws-library/requirements/_base.in -types-aiobotocore-ec2==2.15.2 +types-aiobotocore-ec2==2.13.0 # via types-aiobotocore -types-aiobotocore-s3==2.15.2 +types-aiobotocore-s3==2.13.0 # via types-aiobotocore -types-aiobotocore-ssm==2.15.2 +types-aiobotocore-ssm==2.13.1 # via types-aiobotocore -types-awscrt==0.22.0 +types-awscrt==0.20.9 # via botocore-stubs -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20240316 # via arrow -typing-extensions==4.12.2 +typing-extensions==4.11.0 # via # aiodebug + # aiodocker # fastapi # faststream # opentelemetry-sdk @@ -465,7 +454,7 @@ typing-extensions==4.12.2 # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm -urllib3==2.2.3 +urllib3==2.2.1 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -481,19 +470,17 @@ urllib3==2.2.3 # -c requirements/../../../requirements/constraints.txt # botocore # requests -uvicorn==0.31.1 +uvicorn==0.30.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in wrapt==1.16.0 # via # aiobotocore # deprecated # opentelemetry-instrumentation -wsproto==1.2.0 - # via simple-websocket -yarl==1.14.0 +yarl==1.9.4 # via # aio-pika # aiohttp # aiormq -zipp==3.20.2 +zipp==3.20.1 # via importlib-metadata diff --git a/services/efs-guardian/requirements/_test.txt b/services/efs-guardian/requirements/_test.txt index 3c576c58c46..efd05c557f6 100644 --- a/services/efs-guardian/requirements/_test.txt +++ b/services/efs-guardian/requirements/_test.txt @@ -1,12 +1,8 @@ -aiodocker==0.23.0 +aiodocker==0.21.0 # via # -c requirements/_base.txt # -r requirements/_test.in -aiohappyeyeballs==2.4.3 - # via - # -c requirements/_base.txt - # aiohttp -aiohttp==3.10.10 +aiohttp==3.9.5 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -17,37 +13,37 @@ aiosignal==1.3.1 # aiohttp antlr4-python3-runtime==4.13.2 # via moto -anyio==4.6.0 +anyio==4.4.0 # via # -c requirements/_base.txt # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -attrs==24.2.0 +attrs==23.2.0 # via # -c requirements/_base.txt # aiohttp # jsonschema # referencing -aws-sam-translator==1.91.0 +aws-sam-translator==1.89.0 # via cfn-lint aws-xray-sdk==2.14.0 # via moto blinker==1.8.2 # via flask -boto3==1.34.131 +boto3==1.34.106 # via # -c requirements/_base.txt # aws-sam-translator # moto -botocore==1.34.131 +botocore==1.34.106 # via # -c requirements/_base.txt # aws-xray-sdk # boto3 # moto # s3transfer -certifi==2024.8.30 +certifi==2024.2.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -56,9 +52,9 @@ certifi==2024.8.30 # requests cffi==1.17.1 # via cryptography -cfn-lint==1.16.1 +cfn-lint==1.10.3 # via moto -charset-normalizer==3.4.0 +charset-normalizer==3.3.2 # via # -c requirements/_base.txt # requests @@ -66,7 +62,7 @@ click==8.1.7 # via # -c requirements/_base.txt # flask -coverage==7.6.2 +coverage==7.6.1 # via # -r requirements/_test.in # pytest-cov @@ -75,7 +71,7 @@ cryptography==43.0.1 # -c requirements/../../../requirements/constraints.txt # joserfc # moto -debugpy==1.8.7 +debugpy==1.8.5 # via -r requirements/_test.in deepdiff==8.0.1 # via -r requirements/_test.in @@ -83,9 +79,9 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -faker==30.3.0 +faker==29.0.0 # via -r requirements/_test.in -fakeredis==2.25.1 +fakeredis==2.24.1 # via -r requirements/_test.in flask==3.0.3 # via @@ -104,17 +100,17 @@ h11==0.14.0 # via # -c requirements/_base.txt # httpcore -httpcore==1.0.6 +httpcore==1.0.5 # via # -c requirements/_base.txt # httpx -httpx==0.27.2 +httpx==0.27.0 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -r requirements/_test.in # respx -idna==3.10 +idna==3.7 # via # -c requirements/_base.txt # anyio @@ -145,7 +141,7 @@ jsonpath-ng==1.6.1 # via moto jsonpointer==3.0.0 # via jsonpatch -jsonschema==4.23.0 +jsonschema==4.22.0 # via # -c requirements/_base.txt # aws-sam-translator @@ -162,20 +158,20 @@ lazy-object-proxy==1.10.0 # via openapi-spec-validator lupa==2.2 # via fakeredis -markupsafe==3.0.1 +markupsafe==2.1.5 # via # jinja2 # werkzeug -moto==5.0.16 +moto==5.0.15 # via -r requirements/_test.in mpmath==1.3.0 # via sympy -multidict==6.1.0 +multidict==6.0.5 # via # -c requirements/_base.txt # aiohttp # yarl -networkx==3.4 +networkx==3.3 # via cfn-lint openapi-schema-validator==0.6.2 # via openapi-spec-validator @@ -183,7 +179,7 @@ openapi-spec-validator==0.7.1 # via moto orderly-set==5.2.2 # via deepdiff -packaging==24.1 +packaging==24.0 # via # -c requirements/_base.txt # pytest @@ -195,10 +191,6 @@ pluggy==1.5.0 # via pytest ply==3.11 # via jsonpath-ng -propcache==0.2.0 - # via - # -c requirements/_base.txt - # yarl psutil==6.0.0 # via # -c requirements/_base.txt @@ -207,7 +199,7 @@ py-partiql-parser==0.5.6 # via moto pycparser==2.22 # via cffi -pydantic==1.10.18 +pydantic==1.10.15 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -238,7 +230,7 @@ python-dateutil==2.9.0.post0 # moto python-dotenv==1.0.1 # via -r requirements/_test.in -pyyaml==6.0.2 +pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -247,7 +239,7 @@ pyyaml==6.0.2 # jsonschema-path # moto # responses -redis==5.1.1 +redis==5.0.4 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -273,16 +265,16 @@ respx==0.21.1 # via -r requirements/_test.in rfc3339-validator==0.1.4 # via openapi-schema-validator -rpds-py==0.20.0 +rpds-py==0.18.1 # via # -c requirements/_base.txt # jsonschema # referencing -s3transfer==0.10.3 +s3transfer==0.10.1 # via # -c requirements/_base.txt # boto3 -setuptools==75.1.0 +setuptools==74.0.0 # via # -c requirements/_base.txt # moto @@ -301,14 +293,14 @@ sortedcontainers==2.4.0 # via fakeredis sympy==1.13.3 # via cfn-lint -typing-extensions==4.12.2 +typing-extensions==4.11.0 # via # -c requirements/_base.txt + # aiodocker # aws-sam-translator # cfn-lint - # faker # pydantic -urllib3==2.2.3 +urllib3==2.2.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt @@ -324,9 +316,9 @@ wrapt==1.16.0 # via # -c requirements/_base.txt # aws-xray-sdk -xmltodict==0.14.1 +xmltodict==0.13.0 # via moto -yarl==1.14.0 +yarl==1.9.4 # via # -c requirements/_base.txt # aiohttp diff --git a/services/efs-guardian/requirements/_tools.txt b/services/efs-guardian/requirements/_tools.txt index 8eac6b4208e..97a49efc2eb 100644 --- a/services/efs-guardian/requirements/_tools.txt +++ b/services/efs-guardian/requirements/_tools.txt @@ -1,8 +1,8 @@ -astroid==3.3.5 +astroid==3.3.4 # via pylint -black==24.10.0 +black==24.8.0 # via -r requirements/../../../requirements/devenv.txt -build==1.2.2.post1 +build==1.2.2 # via pip-tools bump2version==1.0.1 # via -r requirements/../../../requirements/devenv.txt @@ -14,9 +14,9 @@ click==8.1.7 # -c requirements/_test.txt # black # pip-tools -dill==0.3.9 +dill==0.3.8 # via pylint -distlib==0.3.9 +distlib==0.3.8 # via virtualenv filelock==3.16.1 # via virtualenv @@ -28,7 +28,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.12.0 +mypy==1.11.2 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via @@ -36,7 +36,7 @@ mypy-extensions==1.0.0 # mypy nodeenv==1.9.1 # via pre-commit -packaging==24.1 +packaging==24.0 # via # -c requirements/_base.txt # -c requirements/_test.txt @@ -53,38 +53,38 @@ platformdirs==4.3.6 # black # pylint # virtualenv -pre-commit==4.0.1 +pre-commit==3.8.0 # via -r requirements/../../../requirements/devenv.txt -pylint==3.3.1 +pylint==3.3.0 # via -r requirements/../../../requirements/devenv.txt -pyproject-hooks==1.2.0 +pyproject-hooks==1.1.0 # via # build # pip-tools -pyyaml==6.0.2 +pyyaml==6.0.1 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # -c requirements/_test.txt # pre-commit # watchdog -ruff==0.6.9 +ruff==0.6.7 # via -r requirements/../../../requirements/devenv.txt -setuptools==75.1.0 +setuptools==74.0.0 # via # -c requirements/_base.txt # -c requirements/_test.txt # pip-tools tomlkit==0.13.2 # via pylint -typing-extensions==4.12.2 +typing-extensions==4.11.0 # via # -c requirements/_base.txt # -c requirements/_test.txt # mypy -virtualenv==20.26.6 +virtualenv==20.26.5 # via pre-commit -watchdog==5.0.3 +watchdog==5.0.2 # via -r requirements/_tools.in wheel==0.44.0 # via pip-tools diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py index 46c14496ee9..6bf2833ed02 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/core/application.py @@ -17,7 +17,6 @@ from ..services.efs_manager_setup import setup as setup_efs_manager from ..services.modules.rabbitmq import setup as setup_rabbitmq from ..services.modules.redis import setup as setup_redis -from ..services.notifications._setup import setup_notifications from ..services.process_messages_setup import setup as setup_process_messages from .settings import ApplicationSettings @@ -53,8 +52,6 @@ def create_app(settings: ApplicationSettings) -> FastAPI: setup_background_tasks(app) setup_process_messages(app) - setup_notifications(app) - # EVENTS async def _on_startup() -> None: print(APP_STARTED_BANNER_MSG, flush=True) # noqa: T201 diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py index be0460b7e64..37dca6afe70 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/efs_manager.py @@ -1,3 +1,4 @@ +import logging import os from dataclasses import dataclass from pathlib import Path @@ -10,6 +11,8 @@ from ..core.settings import ApplicationSettings, get_application_settings from . import efs_manager_utils +_logger = logging.getLogger(__name__) + @dataclass(frozen=True) class EfsManager: @@ -79,6 +82,31 @@ async def get_project_node_data_size( return await efs_manager_utils.get_size_bash_async(_dir_path) + async def list_project_node_state_names( + self, project_id: ProjectID, node_id: NodeID + ) -> list[str]: + """ + These are currently state volumes that are mounted via docker volume to dynamic sidecar and user services + (ex. ".data_assets" and "home_user_workspace") + """ + _dir_path = ( + self._efs_mounted_path + / self._project_specific_data_base_directory + / f"{project_id}" + / f"{node_id}" + ) + + project_node_states = [] + for child in _dir_path.iterdir(): + if child.is_dir(): + project_node_states.append(child.name) + else: + _logger.error( + "This is not a directory. This should not happen! %s", + _dir_path / child.name, + ) + return project_node_states + async def remove_project_node_data_write_permissions( self, project_id: ProjectID, node_id: NodeID ) -> None: diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py index 82ef1aae84c..f94c5dbf418 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/modules/rabbitmq.py @@ -29,12 +29,17 @@ async def on_startup() -> None: app.state.rabbitmq_rpc_server = await RabbitMQRPCClient.create( client_name="efs_guardian_rpc_server", settings=settings ) + app.state.rabbitmq_rpc_client = await RabbitMQRPCClient.create( + client_name="efs_guardian_rpc_client", settings=settings + ) async def on_shutdown() -> None: if app.state.rabbitmq_client: await app.state.rabbitmq_client.close() if app.state.rabbitmq_rpc_server: await app.state.rabbitmq_rpc_server.close() + if app.state.rabbitmq_rpc_client: + await app.state.rabbitmq_rpc_client.close() app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) @@ -53,4 +58,9 @@ def get_rabbitmq_rpc_server(app: FastAPI) -> RabbitMQRPCClient: return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_server) +def get_rabbitmq_rpc_client(app: FastAPI) -> RabbitMQRPCClient: + assert app.state.rabbitmq_rpc_client # nosec + return cast(RabbitMQRPCClient, app.state.rabbitmq_rpc_client) + + __all__ = ("RabbitMQClient",) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py deleted file mode 100644 index d270853f5a7..00000000000 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from ._setup import setup_notifications - -__all__: tuple[str, ...] = ("setup_notifications",) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py deleted file mode 100644 index a96ec1abc83..00000000000 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_notifier.py +++ /dev/null @@ -1,59 +0,0 @@ -import contextlib -from typing import Final - -import socketio # type: ignore[import-untyped] -from fastapi import FastAPI -from fastapi.encoders import jsonable_encoder -from models_library.api_schemas_webserver.socketio import SocketIORoomStr -from models_library.projects_nodes_io import NodeID -from models_library.users import UserID -from pydantic import BaseModel, ByteSize, Field -from servicelib.fastapi.app_state import SingletonInAppStateMixin - -SOCKET_IO_EFS_DISK_USAGE_EVENT: Final[str] = "efsNodeDiskUsage" - - -class EfsNodeDiskUsage(BaseModel): - node_id: NodeID - used: ByteSize = Field(description="used space") - free: ByteSize = Field(description="remaining space") - total: ByteSize = Field(description="total space = free + used") - used_percent: float = Field( - gte=0.00, - lte=100.00, - description="Percent of used space relative to the total space", - ) - - -class Notifier(SingletonInAppStateMixin): - app_state_name: str = "notifier" - - def __init__(self, sio_manager: socketio.AsyncAioPikaManager): - self._sio_manager = sio_manager - - async def notify_service_efs_disk_usage( - self, user_id: UserID, efs_node_disk_usage: EfsNodeDiskUsage - ) -> None: - await self._sio_manager.emit( - SOCKET_IO_EFS_DISK_USAGE_EVENT, - data=jsonable_encoder(efs_node_disk_usage), - room=SocketIORoomStr.from_user_id(user_id), - ) - - -def setup_notifier(app: FastAPI): - async def _on_startup() -> None: - assert app.state.external_socketio # nosec - - notifier = Notifier( - sio_manager=app.state.external_socketio, - ) - notifier.set_to_app_state(app) - assert Notifier.get_from_app_state(app) == notifier # nosec - - async def _on_shutdown() -> None: - with contextlib.suppress(AttributeError): - Notifier.pop_from_app_state(app) - - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py deleted file mode 100644 index 6de0fae307f..00000000000 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_setup.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging - -from fastapi import FastAPI -from servicelib.logging_utils import log_context - -from ..notifications._notifier import setup_notifier -from ..notifications._socketio import setup_socketio - -_logger = logging.getLogger(__name__) - - -def setup_notifications(app: FastAPI) -> None: - with log_context(_logger, logging.INFO, "setup notifications"): - setup_socketio(app) - setup_notifier(app) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py deleted file mode 100644 index 5ceb1049148..00000000000 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/notifications/_socketio.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging - -import socketio # type: ignore[import-untyped] -from fastapi import FastAPI -from servicelib.socketio_utils import cleanup_socketio_async_pubsub_manager - -from ...core.settings import ApplicationSettings - -_logger = logging.getLogger(__name__) - - -def setup_socketio(app: FastAPI): - settings: ApplicationSettings = app.state.settings - - async def _on_startup() -> None: - assert app.state.rabbitmq_client # nosec - - # Connect to the as an external process in write-only mode - # SEE https://python-socketio.readthedocs.io/en/stable/server.html#emitting-from-external-processes - assert settings.EFS_GUARDIAN_RABBITMQ # nosec - app.state.external_socketio = socketio.AsyncAioPikaManager( - url=settings.EFS_GUARDIAN_RABBITMQ.dsn, logger=_logger, write_only=True - ) - - async def _on_shutdown() -> None: - if external_socketio := getattr(app.state, "external_socketio"): # noqa: B009 - await cleanup_socketio_async_pubsub_manager( - server_manager=external_socketio - ) - - app.add_event_handler("startup", _on_startup) - app.add_event_handler("shutdown", _on_shutdown) diff --git a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py index 2d49fe52c45..8412da67b30 100644 --- a/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py +++ b/services/efs-guardian/src/simcore_service_efs_guardian/services/process_messages.py @@ -1,17 +1,19 @@ import logging from fastapi import FastAPI +from models_library.api_schemas_dynamic_sidecar.telemetry import DiskUsage from models_library.rabbitmq_messages import DynamicServiceRunningMessage -from pydantic import ByteSize, parse_raw_as +from pydantic import parse_raw_as from servicelib.logging_utils import log_context -from simcore_service_efs_guardian.services.modules.redis import get_redis_lock_client -from simcore_service_efs_guardian.services.notifier_setup import ( - EfsNodeDiskUsage, - Notifier, +from servicelib.rabbitmq import RabbitMQRPCClient +from servicelib.rabbitmq.rpc_interfaces.dynamic_sidecar.disk_usage import ( + update_disk_usage, ) from ..core.settings import get_application_settings from ..services.efs_manager import EfsManager +from ..services.modules.rabbitmq import get_rabbitmq_rpc_client +from ..services.modules.redis import get_redis_lock_client _logger = logging.getLogger(__name__) @@ -53,18 +55,23 @@ async def process_dynamic_service_running_message(app: FastAPI, data: bytes) -> rabbit_message.node_id, rabbit_message.user_id, ) - percentage = round(size / settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES * 100, 2) - efs_node_disk_usage = EfsNodeDiskUsage( - node_id=rabbit_message.node_id, - used=size, - free=ByteSize(max(settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES - size, 0)), - total=settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES, - used_percent=min(percentage, 100.0), - ) - notifier: Notifier = Notifier.get_from_app_state(app) - await notifier.notify_service_efs_disk_usage( - user_id=rabbit_message.user_id, efs_node_disk_usage=efs_node_disk_usage + + project_node_state_names = await efs_manager.list_project_node_state_names( + rabbit_message.project_id, node_id=rabbit_message.node_id ) + rpc_client: RabbitMQRPCClient = get_rabbitmq_rpc_client(app) + _used = min(size, settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES) + usage: dict[str, DiskUsage] = {} + for name in project_node_state_names: + usage[name] = DiskUsage.from_efs_guardian( + used=_used, total=settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES + ) + + # usage = { + # ".data_assets": DiskUsage.from_efs_guardian(used=_used, total=settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES), + # "home_user_workspace": DiskUsage.from_efs_guardian(used=_used, total=settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES) + # } + await update_disk_usage(rpc_client, usage=usage) if size > settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES: msg = f"Removing write permissions inside of EFS starts for project ID: {rabbit_message.project_id}, node ID: {rabbit_message.node_id}, current user: {rabbit_message.user_id}, size: {size}, upper limit: {settings.EFS_DEFAULT_USER_SERVICE_SIZE_BYTES}" diff --git a/services/efs-guardian/tests/unit/test_process_messages.py b/services/efs-guardian/tests/unit/test_process_messages.py index facaf9380aa..615e2204273 100644 --- a/services/efs-guardian/tests/unit/test_process_messages.py +++ b/services/efs-guardian/tests/unit/test_process_messages.py @@ -40,11 +40,9 @@ def app_environment( ) -@patch( - "simcore_service_efs_guardian.services.process_messages.Notifier.get_from_app_state" -) +@patch("simcore_service_efs_guardian.services.process_messages.update_disk_usage") async def test_process_msg( - mock_notifier_get_from_app_state, + mock_update_disk_usage, faker: Faker, mocked_redis_server: None, app: FastAPI, @@ -64,35 +62,32 @@ async def test_process_msg( json_str = model_instance.json() model_bytes = json_str.encode("utf-8") + _expected_project_node_states = [".data_assets", "home_user_workspace"] # Mock efs_manager and its methods mock_efs_manager = AsyncMock() app.state.efs_manager = mock_efs_manager mock_efs_manager.check_project_node_data_directory_exits.return_value = True mock_efs_manager.get_project_node_data_size.return_value = 4000 - - # Mock the Notifier - mock_notifier = AsyncMock() - mock_notifier_get_from_app_state.return_value = mock_notifier + mock_efs_manager.list_project_node_state_names.return_value = ( + _expected_project_node_states + ) result = await process_dynamic_service_running_message(app, data=model_bytes) # Check the actual arguments passed to notify_service_efs_disk_usage - _, kwargs = mock_notifier.notify_service_efs_disk_usage.call_args - assert kwargs["user_id"] == user_id - efs_node_disk_usage = kwargs["efs_node_disk_usage"] - - # Check EfsNodeDiskUsage values - assert efs_node_disk_usage.node_id == node_id - assert efs_node_disk_usage.used == 4000 - assert efs_node_disk_usage.free == 6000 - assert efs_node_disk_usage.total == 10000 - assert efs_node_disk_usage.used_percent == 40.0 + _, kwargs = mock_update_disk_usage.call_args + assert kwargs["usage"] + assert len(kwargs["usage"]) == 2 + for key, value in kwargs["usage"].items(): + assert key in _expected_project_node_states + assert value.used == 4000 + assert value.free == 6000 + assert value.total == 10000 + assert value.used_percent == 40.0 - # Check that the function returns True assert result is True -@pytest.mark.skip() async def test_process_msg__dir_not_exists( mocked_redis_server: None, app: FastAPI, From aa91db3b30900cd204e2a99e32da6033321dbf2f Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 22 Oct 2024 13:13:40 +0200 Subject: [PATCH 6/8] fix mypy version --- services/efs-guardian/requirements/_tools.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/efs-guardian/requirements/_tools.txt b/services/efs-guardian/requirements/_tools.txt index 97a49efc2eb..dec3b9c204d 100644 --- a/services/efs-guardian/requirements/_tools.txt +++ b/services/efs-guardian/requirements/_tools.txt @@ -28,7 +28,7 @@ isort==5.13.2 # pylint mccabe==0.7.0 # via pylint -mypy==1.11.2 +mypy==1.12.0 # via -r requirements/../../../requirements/devenv.txt mypy-extensions==1.0.0 # via From fe077b97bf11e5c6988a85bac0482b6c93ac3a81 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 22 Oct 2024 15:34:29 +0200 Subject: [PATCH 7/8] improve test --- services/efs-guardian/tests/unit/test_efs_manager.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py index 5c5c57cf3ab..ee997ec4bd0 100644 --- a/services/efs-guardian/tests/unit/test_efs_manager.py +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -90,6 +90,11 @@ async def test_remove_write_access_rights( is False ) + project_node_state_names = await efs_manager.list_project_node_state_names( + project_id=project_id, node_id=node_id + ) + assert project_node_state_names == [] + with patch( "simcore_service_efs_guardian.services.efs_manager.os.chown" ) as mocked_chown: @@ -107,6 +112,11 @@ async def test_remove_write_access_rights( is True ) + project_node_state_names = await efs_manager.list_project_node_state_names( + project_id=project_id, node_id=node_id + ) + assert project_node_state_names == [_storage_directory_name] + size_before = await efs_manager.get_project_node_data_size( project_id=project_id, node_id=node_id ) From f327653be7ca5a18a6deeb1a2214babd8e7d84a6 Mon Sep 17 00:00:00 2001 From: matusdrobuliak66 Date: Tue, 29 Oct 2024 21:03:23 +0100 Subject: [PATCH 8/8] fix unit test --- services/efs-guardian/tests/unit/test_efs_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/efs-guardian/tests/unit/test_efs_manager.py b/services/efs-guardian/tests/unit/test_efs_manager.py index 18ab13d14ce..35c2535de94 100644 --- a/services/efs-guardian/tests/unit/test_efs_manager.py +++ b/services/efs-guardian/tests/unit/test_efs_manager.py @@ -91,10 +91,10 @@ async def test_remove_write_access_rights( is False ) - project_node_state_names = await efs_manager.list_project_node_state_names( - project_id=project_id, node_id=node_id - ) - assert project_node_state_names == [] + with pytest.raises(FileNotFoundError): + await efs_manager.list_project_node_state_names( + project_id=project_id, node_id=node_id + ) with patch( "simcore_service_efs_guardian.services.efs_manager.os.chown"