From bd6a8be15766046ca4547ebd3e439a8a83cab7b4 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Fri, 11 Oct 2024 09:59:45 +0200 Subject: [PATCH 01/22] upgrade reqs --- .../requirements/_base.txt | 57 ++++++++++--------- .../requirements/_test.txt | 31 ++++------ .../requirements/_tools.txt | 10 ---- .../core/settings.py | 32 +++++++---- .../resource_tracker_credit_transactions.py | 6 +- .../models/resource_tracker_pricing_plans.py | 14 ++--- .../resource_tracker_pricing_unit_costs.py | 6 +- .../models/resource_tracker_pricing_units.py | 8 +-- .../models/resource_tracker_service_runs.py | 20 ++----- .../resource_tracker_pricing_plans.py | 4 +- .../tests/unit/api_rest/test_api_meta.py | 2 +- ...test_api_resource_tracker_pricing_plans.py | 16 ++++-- ..._api_resource_tracker_pricing_plans_rpc.py | 16 ++++-- ...age_triggered_by_listening_with_billing.py | 12 +++- ...t_process_rabbitmq_message_with_billing.py | 12 +++- ...ss_rabbitmq_message_with_billing_cost_0.py | 4 +- 16 files changed, 127 insertions(+), 123 deletions(-) diff --git a/services/resource-usage-tracker/requirements/_base.txt b/services/resource-usage-tracker/requirements/_base.txt index 64c4ae399ad..8540bb3da25 100644 --- a/services/resource-usage-tracker/requirements/_base.txt +++ b/services/resource-usage-tracker/requirements/_base.txt @@ -48,6 +48,8 @@ aiosignal==1.3.1 # via aiohttp alembic==1.13.1 # via -r requirements/../../../packages/postgres-database/requirements/_base.in +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # fast-depends @@ -60,14 +62,12 @@ arrow==1.3.0 # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/_base.in # -r requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in # -r requirements/../../../packages/service-library/requirements/_base.in async-timeout==4.0.3 - # via - # aiohttp - # asyncpg - # redis + # via asyncpg asyncpg==0.29.0 # via sqlalchemy attrs==23.2.0 @@ -118,25 +118,10 @@ dnspython==2.6.1 # via email-validator email-validator==2.1.1 # via pydantic -exceptiongroup==1.2.0 - # via anyio fast-depends==2.4.2 # via faststream -fastapi==0.99.1 +fastapi==0.115.0 # via - # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/aws-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/service-library/requirements/_fastapi.in # -r requirements/_base.in # prometheus-fastapi-instrumentator @@ -272,7 +257,7 @@ prometheus-fastapi-instrumentator==6.1.0 # via -r requirements/../../../packages/service-library/requirements/_fastapi.in psycopg2-binary==2.9.9 # via sqlalchemy -pydantic==1.10.14 +pydantic==2.9.2 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -284,7 +269,6 @@ pydantic==1.10.14 # -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt - # -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in # -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../requirements/constraints.txt @@ -302,6 +286,26 @@ pydantic==1.10.14 # -r requirements/../../../packages/settings-library/requirements/_base.in # fast-depends # fastapi + # pydantic-extra-types + # pydantic-settings +pydantic-core==2.23.4 + # via pydantic +pydantic-extra-types==2.9.0 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in +pydantic-settings==2.5.2 + # via + # -r requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/aws-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in + # -r requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/_base.in + # -r requirements/../../../packages/settings-library/requirements/_base.in pygments==2.17.2 # via rich pyinstrument==4.6.2 @@ -318,7 +322,9 @@ python-dateutil==2.9.0.post0 # matplotlib # pandas python-dotenv==1.0.1 - # via uvicorn + # via + # pydantic-settings + # uvicorn pytz==2024.1 # via # dateparser @@ -412,7 +418,7 @@ sqlalchemy==1.4.52 # -c requirements/../../../requirements/constraints.txt # -r requirements/../../../packages/postgres-database/requirements/_base.in # alembic -starlette==0.27.0 +starlette==0.38.6 # via # -c requirements/../../../packages/aws-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt # -c requirements/../../../packages/aws-library/requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt @@ -465,16 +471,15 @@ typing-extensions==4.10.0 # aiodebug # aiodocker # alembic - # anyio # fastapi # faststream # pydantic + # pydantic-core # typer # types-aiobotocore # types-aiobotocore-ec2 # types-aiobotocore-s3 # types-aiobotocore-ssm - # uvicorn tzdata==2024.1 # via pandas tzlocal==5.2 diff --git a/services/resource-usage-tracker/requirements/_test.txt b/services/resource-usage-tracker/requirements/_test.txt index de56ac958c7..3ff748009f7 100644 --- a/services/resource-usage-tracker/requirements/_test.txt +++ b/services/resource-usage-tracker/requirements/_test.txt @@ -2,6 +2,10 @@ alembic==1.13.1 # via # -c requirements/_base.txt # -r requirements/_test.in +annotated-types==0.7.0 + # via + # -c requirements/_base.txt + # pydantic antlr4-python3-runtime==4.13.2 # via moto anyio==4.3.0 @@ -10,10 +14,6 @@ anyio==4.3.0 # httpx asgi-lifespan==2.1.0 # via -r requirements/_test.in -async-timeout==4.0.3 - # via - # -c requirements/_base.txt - # redis attrs==23.2.0 # via # -c requirements/_base.txt @@ -69,11 +69,6 @@ docker==7.1.0 # via # -r requirements/_test.in # moto -exceptiongroup==1.2.0 - # via - # -c requirements/_base.txt - # anyio - # pytest faker==27.0.0 # via -r requirements/_test.in fakeredis==2.23.5 @@ -86,10 +81,6 @@ flask-cors==4.0.1 # via moto graphql-core==3.2.3 # via moto -greenlet==3.0.3 - # via - # -c requirements/_base.txt - # sqlalchemy h11==0.14.0 # via # -c requirements/_base.txt @@ -191,11 +182,15 @@ py-partiql-parser==0.5.5 # via moto pycparser==2.22 # via cffi -pydantic==1.10.14 +pydantic==2.9.2 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt # aws-sam-translator +pydantic-core==2.23.4 + # via + # -c requirements/_base.txt + # pydantic pyparsing==3.1.2 # via # -c requirements/_base.txt @@ -305,23 +300,17 @@ sympy==1.13.2 # via cfn-lint termcolor==2.4.0 # via pytest-sugar -tomli==2.0.1 - # via - # coverage - # mypy - # pytest types-requests==2.32.0.20240712 # via -r requirements/_test.in typing-extensions==4.10.0 # via # -c requirements/_base.txt # alembic - # anyio # aws-sam-translator # cfn-lint - # fakeredis # mypy # pydantic + # pydantic-core # sqlalchemy2-stubs urllib3==2.0.7 # via diff --git a/services/resource-usage-tracker/requirements/_tools.txt b/services/resource-usage-tracker/requirements/_tools.txt index b84cbbeb9fb..9ea4ddf842c 100644 --- a/services/resource-usage-tracker/requirements/_tools.txt +++ b/services/resource-usage-tracker/requirements/_tools.txt @@ -77,22 +77,12 @@ setuptools==73.0.1 # via # -c requirements/_test.txt # pip-tools -tomli==2.0.1 - # via - # -c requirements/_test.txt - # black - # build - # mypy - # pip-tools - # pylint tomlkit==0.13.2 # via pylint typing-extensions==4.10.0 # via # -c requirements/_base.txt # -c requirements/_test.txt - # astroid - # black # mypy virtualenv==20.26.3 # via pre-commit diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 7ac7599ed58..360a811dd86 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -2,7 +2,7 @@ from functools import cached_property from models_library.basic_types import BootModeEnum -from pydantic import Field, PositiveInt, validator +from pydantic import AliasChoices, field_validator, Field, PositiveInt from settings_library.base import BaseCustomSettings from settings_library.basic_types import BuildTargetEnum, LogLevel, VersionTag from settings_library.postgres import PostgresSettings @@ -44,18 +44,24 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): RESOURCE_USAGE_TRACKER_DEBUG: bool = Field( default=False, description="Debug mode", - env=["RESOURCE_USAGE_TRACKER_DEBUG", "DEBUG"], + validation_alias=AliasChoices( + "RESOURCE_USAGE_TRACKER_DEBUG", + "DEBUG", + ), ) RESOURCE_USAGE_TRACKER_LOGLEVEL: LogLevel = Field( default=LogLevel.INFO, - env=["RESOURCE_USAGE_TRACKER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"], + validation_alias=AliasChoices( + "RESOURCE_USAGE_TRACKER_LOGLEVEL", + "LOG_LEVEL", + "LOGLEVEL"), ) RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, - env=[ + validation_alias=AliasChoices( "RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", - ], + ), description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ) @@ -63,7 +69,7 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): def LOG_LEVEL(self) -> LogLevel: # noqa: N802 return self.RESOURCE_USAGE_TRACKER_LOGLEVEL - @validator("RESOURCE_USAGE_TRACKER_LOGLEVEL") + @field_validator("RESOURCE_USAGE_TRACKER_LOGLEVEL") @classmethod def valid_log_level(cls, value: str) -> str: return cls.validate_log_level(value) @@ -77,16 +83,18 @@ class MinimalApplicationSettings(_BaseApplicationSettings): """ RESOURCE_USAGE_TRACKER_PROMETHEUS: PrometheusSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env":True} ) RESOURCE_USAGE_TRACKER_POSTGRES: PostgresSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env":True}, ) - RESOURCE_USAGE_TRACKER_REDIS: RedisSettings = Field(auto_default_from_env=True) + RESOURCE_USAGE_TRACKER_REDIS: RedisSettings = Field( + json_schema_extra={"auto_default_from_env":True}, + ) RESOURCE_USAGE_TRACKER_RABBITMQ: RabbitSettings | None = Field( - auto_default_from_env=True + json_schema_extra={"auto_default_from_env":True}, ) @@ -109,4 +117,6 @@ class ApplicationSettings(MinimalApplicationSettings): description="Heartbeat couter limit when RUT considers service as unhealthy.", ) RESOURCE_USAGE_TRACKER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True - RESOURCE_USAGE_TRACKER_S3: S3Settings | None = Field(auto_default_from_env=True) + RESOURCE_USAGE_TRACKER_S3: S3Settings | None = Field( + json_schema_extra={"auto_default_from_env":True}, + ) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py index a264f90d375..88c49c10713 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_credit_transactions.py @@ -13,7 +13,7 @@ ) from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel +from pydantic import ConfigDict, BaseModel class CreditTransactionCreate(BaseModel): @@ -64,6 +64,4 @@ class CreditTransactionDB(BaseModel): created: datetime last_heartbeat_at: datetime modified: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py index f946c92e5d9..1c17a433855 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_plans.py @@ -2,7 +2,7 @@ from models_library.resource_tracker import PricingPlanClassification, PricingPlanId from models_library.services import ServiceKey, ServiceVersion -from pydantic import BaseModel +from pydantic import ConfigDict, BaseModel ## DB Models @@ -15,16 +15,12 @@ class PricingPlansDB(BaseModel): is_active: bool created: datetime pricing_plan_key: str - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class PricingPlansWithServiceDefaultPlanDB(PricingPlansDB): service_default_plan: bool - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class PricingPlanToServiceDB(BaseModel): @@ -32,6 +28,4 @@ class PricingPlanToServiceDB(BaseModel): service_key: ServiceKey service_version: ServiceVersion created: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py index b5fa3daadf0..23fb94cf7cf 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_unit_costs.py @@ -6,7 +6,7 @@ PricingUnitCostId, PricingUnitId, ) -from pydantic import BaseModel +from pydantic import ConfigDict, BaseModel class PricingUnitCostsDB(BaseModel): @@ -21,6 +21,4 @@ class PricingUnitCostsDB(BaseModel): created: datetime comment: str | None modified: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py index f0fed877d43..5a0a66dc542 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_pricing_units.py @@ -9,7 +9,7 @@ PricingUnitId, UnitExtraInfo, ) -from pydantic import BaseModel, validator +from pydantic import field_validator, ConfigDict, BaseModel class PricingUnitsDB(BaseModel): @@ -23,11 +23,9 @@ class PricingUnitsDB(BaseModel): modified: datetime current_cost_per_unit: Decimal current_cost_per_unit_id: PricingUnitCostId + model_config = ConfigDict(from_attributes=True) - class Config: - orm_mode = True - - @validator("specific_info", pre=True) + @field_validator("specific_info", mode="before") @classmethod def default_hardware_info_when_empty(cls, v) -> HardwareInfo | Any: if not v: diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py index 45cddca3057..011da17901f 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/models/resource_tracker_service_runs.py @@ -16,7 +16,7 @@ from models_library.services import ServiceKey, ServiceVersion from models_library.users import UserID from models_library.wallets import WalletID -from pydantic import BaseModel, NonNegativeInt +from pydantic import ConfigDict, BaseModel, NonNegativeInt class ServiceRunCreate(BaseModel): @@ -93,25 +93,19 @@ class ServiceRunDB(BaseModel): last_heartbeat_at: datetime service_run_status_msg: str | None missed_heartbeat_counter: NonNegativeInt - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class ServiceRunWithCreditsDB(ServiceRunDB): - osparc_credits: Decimal | None + osparc_credits: Decimal | None = None transaction_status: CreditTransactionStatus | None - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class OsparcCreditsAggregatedByServiceKeyDB(BaseModel): osparc_credits: Decimal service_key: ServiceKey - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class ServiceRunForCheckDB(BaseModel): @@ -119,6 +113,4 @@ class ServiceRunForCheckDB(BaseModel): last_heartbeat_at: datetime missed_heartbeat_counter: NonNegativeInt modified: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py index d37f244dbc9..faf7335e1ac 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py @@ -96,7 +96,7 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ] = await resource_tracker_repo.list_connected_services_to_pricing_plan_by_pricing_plan( product_name=product_name, pricing_plan_id=pricing_plan_id ) - return [PricingPlanToServiceGet.parse_obj(item) for item in output_list] + return [PricingPlanToServiceGet.model_validate(item) for item in output_list] async def connect_service_to_pricing_plan( @@ -116,7 +116,7 @@ async def connect_service_to_pricing_plan( service_version=service_version, ) ) - return PricingPlanToServiceGet.parse_obj(output) + return PricingPlanToServiceGet.model_validate(output) async def list_pricing_plans_by_product( diff --git a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py index cffb606fae5..568abc8eca0 100644 --- a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py +++ b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py @@ -68,7 +68,7 @@ def test_meta( ): response = client.get(f"/{API_VTAG}/meta") assert response.status_code == status.HTTP_200_OK - meta = _Meta.parse_obj(response.json()) + meta = _Meta.model_validate(response.json()) response = client.get(meta.docs_url) assert response.status_code == status.HTTP_200_OK diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py index 5e241e60767..b8c056e732a 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans.py @@ -75,7 +75,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -100,7 +102,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID, unit_name="M", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -125,7 +129,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID, unit_name="L", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -164,7 +170,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=_PRICING_PLAN_ID_2, unit_name="XXL", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py index 5a12fd24dbe..410b1be2e40 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_pricing_plans_rpc.py @@ -160,7 +160,9 @@ async def test_rpc_pricing_plans_with_units_workflow( data=PricingUnitWithCostCreate( pricing_plan_id=_pricing_plan_id, unit_name="SMALL", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=True, specific_info=SpecificInfo(aws_ec2_instances=[]), cost_per_unit=Decimal(10), @@ -192,7 +194,9 @@ async def test_rpc_pricing_plans_with_units_workflow( pricing_plan_id=_pricing_plan_id, pricing_unit_id=_first_pricing_unit_id, unit_name=_unit_name, - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=True, specific_info=SpecificInfo(aws_ec2_instances=[]), pricing_unit_cost_update=None, @@ -211,7 +215,9 @@ async def test_rpc_pricing_plans_with_units_workflow( pricing_plan_id=_pricing_plan_id, pricing_unit_id=_first_pricing_unit_id, unit_name="MEDIUM", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=True, specific_info=SpecificInfo(aws_ec2_instances=[]), pricing_unit_cost_update=PricingUnitCostUpdate( @@ -242,7 +248,9 @@ async def test_rpc_pricing_plans_with_units_workflow( data=PricingUnitWithCostCreate( pricing_plan_id=_pricing_plan_id, unit_name="LARGE", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"]["examples"][ + 0 + ], default=False, specific_info=SpecificInfo(aws_ec2_instances=[]), cost_per_unit=Decimal(20), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py index 8d95ae78d75..b250f0d654f 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_triggered_by_listening_with_billing.py @@ -57,7 +57,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -82,7 +84,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="M", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -107,7 +111,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="L", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py index 92946509e91..eedb6d11c61 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing.py @@ -71,7 +71,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -96,7 +98,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="M", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=True, specific_info={}, created=datetime.now(tz=timezone.utc), @@ -121,7 +125,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="L", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py index d5bc497fb0f..7522d751d55 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_process_rabbitmq_message_with_billing_cost_0.py @@ -67,7 +67,9 @@ def resource_tracker_pricing_tables_db(postgres_db: sa.engine.Engine) -> Iterato resource_tracker_pricing_units.insert().values( pricing_plan_id=1, unit_name="S", - unit_extra_info=UnitExtraInfo.Config.schema_extra["examples"][0], + unit_extra_info=UnitExtraInfo.model_config["json_schema_extra"][ + "examples" + ][0], default=False, specific_info={}, created=datetime.now(tz=timezone.utc), From b7f91f0a2342c8d0a42ef7d4a960e7c3c98f6a3e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Oct 2024 11:16:19 +0200 Subject: [PATCH 02/22] fix code --- .../src/settings_library/base.py | 5 +- services/resource-usage-tracker/openapi.json | 84 +++++++++++-------- .../core/application.py | 2 +- .../core/settings.py | 26 ++++-- .../resource_tracker_process_messages.py | 8 +- .../tests/unit/api_rest/test_api_meta.py | 2 +- 6 files changed, 75 insertions(+), 52 deletions(-) diff --git a/packages/settings-library/src/settings_library/base.py b/packages/settings-library/src/settings_library/base.py index c3f0e103e7a..7bdd7d2cd00 100644 --- a/packages/settings-library/src/settings_library/base.py +++ b/packages/settings-library/src/settings_library/base.py @@ -5,7 +5,7 @@ from common_library.pydantic_fields_extension import get_type, is_literal, is_nullable from pydantic import ValidationInfo, field_validator from pydantic.fields import FieldInfo -from pydantic_core import PydanticUndefined, ValidationError +from pydantic_core import ValidationError from pydantic_settings import BaseSettings, SettingsConfigDict _logger = logging.getLogger(__name__) @@ -102,9 +102,6 @@ def __pydantic_init_subclass__(cls, **kwargs: Any): and issubclass(field_type, BaseCustomSettings) ): if auto_default_from_env: - assert field.default is PydanticUndefined - assert field.default_factory is None - # Transform it into something like `Field(default_factory=create_settings_from_env(field))` field.default_factory = _create_settings_from_env(name, field) field.default = None diff --git a/services/resource-usage-tracker/openapi.json b/services/resource-usage-tracker/openapi.json index 0df986b36a5..c7fe85b4297 100644 --- a/services/resource-usage-tracker/openapi.json +++ b/services/resource-usage-tracker/openapi.json @@ -54,24 +54,24 @@ "operationId": "get_credit_transactions_sum_v1_credit_transactions_credits_sum_post", "parameters": [ { + "name": "product_name", + "in": "query", "required": true, "schema": { "type": "string", "title": "Product Name" - }, - "name": "product_name", - "in": "query" + } }, { + "name": "wallet_id", + "in": "query", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "Wallet Id", "minimum": 0 - }, - "name": "wallet_id", - "in": "query" + } } ], "responses": { @@ -149,33 +149,33 @@ "operationId": "get_service_default_pricing_plan", "parameters": [ { + "name": "service_key", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^simcore/services/((comp|dynamic|frontend))/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$", "title": "Service Key" - }, - "name": "service_key", - "in": "path" + } }, { + "name": "service_version", + "in": "path", "required": true, "schema": { "type": "string", "pattern": "^(0|[1-9]\\d*)(\\.(0|[1-9]\\d*)){2}(-(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*)(\\.(0|[1-9]\\d*|\\d*[-a-zA-Z][-\\da-zA-Z]*))*)?(\\+[-\\da-zA-Z]+(\\.[-\\da-zA-Z-]+)*)?$", "title": "Service Version" - }, - "name": "service_version", - "in": "path" + } }, { + "name": "product_name", + "in": "query", "required": true, "schema": { "type": "string", "title": "Product Name" - }, - "name": "product_name", - "in": "query" + } } ], "responses": { @@ -212,35 +212,35 @@ "operationId": "list_service_pricing_plans", "parameters": [ { + "name": "pricing_plan_id", + "in": "path", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "Pricing Plan Id", "minimum": 0 - }, - "name": "pricing_plan_id", - "in": "path" + } }, { + "name": "pricing_unit_id", + "in": "path", "required": true, "schema": { "type": "integer", "exclusiveMinimum": true, "title": "Pricing Unit Id", "minimum": 0 - }, - "name": "pricing_unit_id", - "in": "path" + } }, { + "name": "product_name", + "in": "query", "required": true, "schema": { "type": "string", "title": "Product Name" - }, - "name": "product_name", - "in": "query" + } } ], "responses": { @@ -297,7 +297,14 @@ "title": "User Email" }, "osparc_credits": { - "type": "number", + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ], "title": "Osparc Credits" }, "payment_transaction_id": { @@ -373,8 +380,8 @@ "enum": [ "TIER" ], - "title": "PricingPlanClassification", - "description": "An enumeration." + "const": "TIER", + "title": "PricingPlanClassification" }, "PricingPlanGet": { "properties": { @@ -405,10 +412,17 @@ "title": "Pricing Plan Key" }, "pricing_units": { - "items": { - "$ref": "#/components/schemas/PricingUnitGet" - }, - "type": "array", + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/PricingUnitGet" + }, + "type": "array" + }, + { + "type": "null" + } + ], "title": "Pricing Units" }, "is_active": { @@ -424,6 +438,7 @@ "classification", "created_at", "pricing_plan_key", + "pricing_units", "is_active" ], "title": "PricingPlanGet" @@ -444,7 +459,7 @@ "$ref": "#/components/schemas/UnitExtraInfo" }, "current_cost_per_unit": { - "type": "number", + "type": "string", "title": "Current Cost Per Unit" }, "current_cost_per_unit_id": { @@ -482,13 +497,16 @@ }, "RAM": { "type": "integer", + "minimum": 0, "title": "Ram" }, "VRAM": { "type": "integer", + "minimum": 0, "title": "Vram" } }, + "additionalProperties": true, "type": "object", "required": [ "CPU", @@ -540,7 +558,7 @@ "minimum": 0 }, "available_osparc_credits": { - "type": "number", + "type": "string", "title": "Available Osparc Credits" } }, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py index d8a696e087e..d8a91085a04 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py @@ -31,7 +31,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: - _logger.info("app settings: %s", settings.json(indent=1)) + _logger.info("app settings: %s", settings.model_dump_json(indent=1)) app = FastAPI( debug=settings.RESOURCE_USAGE_TRACKER_DEBUG, diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index 360a811dd86..94840c72ce3 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -2,7 +2,7 @@ from functools import cached_property from models_library.basic_types import BootModeEnum -from pydantic import AliasChoices, field_validator, Field, PositiveInt +from pydantic import AliasChoices, Field, PositiveInt, field_validator from settings_library.base import BaseCustomSettings from settings_library.basic_types import BuildTargetEnum, LogLevel, VersionTag from settings_library.postgres import PostgresSettings @@ -52,9 +52,8 @@ class _BaseApplicationSettings(BaseCustomSettings, MixinLoggingSettings): RESOURCE_USAGE_TRACKER_LOGLEVEL: LogLevel = Field( default=LogLevel.INFO, validation_alias=AliasChoices( - "RESOURCE_USAGE_TRACKER_LOGLEVEL", - "LOG_LEVEL", - "LOGLEVEL"), + "RESOURCE_USAGE_TRACKER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" + ), ) RESOURCE_USAGE_TRACKER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( default=False, @@ -83,18 +82,18 @@ class MinimalApplicationSettings(_BaseApplicationSettings): """ RESOURCE_USAGE_TRACKER_PROMETHEUS: PrometheusSettings | None = Field( - json_schema_extra={"auto_default_from_env":True} + json_schema_extra={"auto_default_from_env": True} ) RESOURCE_USAGE_TRACKER_POSTGRES: PostgresSettings | None = Field( - json_schema_extra={"auto_default_from_env":True}, + json_schema_extra={"auto_default_from_env": True}, ) RESOURCE_USAGE_TRACKER_REDIS: RedisSettings = Field( - json_schema_extra={"auto_default_from_env":True}, + json_schema_extra={"auto_default_from_env": True}, ) RESOURCE_USAGE_TRACKER_RABBITMQ: RabbitSettings | None = Field( - json_schema_extra={"auto_default_from_env":True}, + json_schema_extra={"auto_default_from_env": True}, ) @@ -118,5 +117,14 @@ class ApplicationSettings(MinimalApplicationSettings): ) RESOURCE_USAGE_TRACKER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True RESOURCE_USAGE_TRACKER_S3: S3Settings | None = Field( - json_schema_extra={"auto_default_from_env":True}, + json_schema_extra={"auto_default_from_env": True}, + ) + + @field_validator( + "RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC", mode="before" ) + @classmethod + def _validate_interval(cls, v): + if isinstance(v, str) and v.isnumeric(): + return int(v) + return v diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py index 4cb024b3b7c..efce3cc4379 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py @@ -20,7 +20,7 @@ ServiceRunStatus, ) from models_library.services import ServiceType -from pydantic import parse_raw_as +from pydantic import TypeAdapter from .models.resource_tracker_credit_transactions import ( CreditTransactionCreate, @@ -45,9 +45,9 @@ async def process_message(app: FastAPI, data: bytes) -> bool: - rabbit_message: RabbitResourceTrackingMessages = parse_raw_as( - RabbitResourceTrackingMessages, data # type: ignore[arg-type] - ) + rabbit_message: RabbitResourceTrackingMessages = TypeAdapter( + RabbitResourceTrackingMessages + ).validate_python(data) _logger.info( "Process %s msg service_run_id: %s", rabbit_message.message_type, diff --git a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py index 568abc8eca0..569ddd28db1 100644 --- a/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py +++ b/services/resource-usage-tracker/tests/unit/api_rest/test_api_meta.py @@ -70,5 +70,5 @@ def test_meta( assert response.status_code == status.HTTP_200_OK meta = _Meta.model_validate(response.json()) - response = client.get(meta.docs_url) + response = client.get(f"{meta.docs_url}") assert response.status_code == status.HTTP_200_OK From 468576ce674c25bae70db7dc52f601d6e847032a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 14 Oct 2024 11:23:43 +0200 Subject: [PATCH 03/22] fix url --- .../api/rpc/_resource_tracker.py | 2 +- .../services/resource_tracker_service_runs.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py index cae70b1152c..94d52f2965a 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/api/rpc/_resource_tracker.py @@ -79,7 +79,7 @@ async def export_service_runs( access_all_wallet_usage: bool = False, order_by: OrderBy | None = None, filters: ServiceResourceUsagesFilters | None = None, -) -> AnyUrl: +) -> str: app_settings: ApplicationSettings = app.state.settings s3_settings = app_settings.RESOURCE_USAGE_TRACKER_S3 assert s3_settings # nosec diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py index 8977d6fa5a0..4b78118365d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_service_runs.py @@ -153,7 +153,7 @@ async def export_service_runs( access_all_wallet_usage: bool = False, order_by: OrderBy | None = None, filters: ServiceResourceUsagesFilters | None = None, -) -> AnyUrl: +) -> str: started_from = filters.started_at.from_ if filters else None started_until = filters.started_at.until if filters else None @@ -177,7 +177,7 @@ async def export_service_runs( ) # Create presigned S3 link - generated_url: AnyUrl = await s3_client.create_single_presigned_download_link( + generated_url: str = await s3_client.create_single_presigned_download_link( bucket=s3_bucket_name, object_key=s3_object_key, expiration_secs=_PRESIGNED_LINK_EXPIRATION_SEC, From 6d95b55d4026d69d665a32bd8cff7363f7b0e620 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Tue, 15 Oct 2024 10:10:25 +0200 Subject: [PATCH 04/22] remove deprecated --- .../src/simcore_service_resource_usage_tracker/_meta.py | 4 ++-- .../test_api_resource_tracker_service_runs__export.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py index 63e86cce819..e792753e07e 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/_meta.py @@ -6,7 +6,7 @@ from models_library.basic_types import VersionStr from packaging.version import Version -from pydantic import parse_obj_as +from pydantic import TypeAdapter from servicelib.utils_meta import PackageInfo from settings_library.basic_types import VersionTag @@ -17,7 +17,7 @@ PROJECT_NAME: Final[str] = info.project_name VERSION: Final[Version] = info.version API_VERSION: Final[VersionStr] = info.__version__ -API_VTAG: Final[VersionTag] = parse_obj_as(VersionTag, info.api_prefix_path_tag) +API_VTAG: Final[VersionTag] = TypeAdapter(VersionTag).validate_python(info.api_prefix_path_tag) SUMMARY: Final[str] = info.get_summary() diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py index c53c1accb90..e433d8c13e6 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_service_runs__export.py @@ -4,7 +4,7 @@ import pytest import sqlalchemy as sa from moto.server import ThreadedMotoServer -from pydantic import AnyUrl, parse_obj_as +from pydantic import AnyUrl, TypeAdapter from pytest_mock import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient @@ -37,8 +37,7 @@ async def mocked_export(mocker: MockerFixture): async def mocked_presigned_link(mocker: MockerFixture): mock_presigned_link = mocker.patch( "simcore_service_resource_usage_tracker.services.resource_tracker_service_runs.SimcoreS3API.create_single_presigned_download_link", - return_value=parse_obj_as( - AnyUrl, + return_value=TypeAdapter(AnyUrl).validate_python( "https://www.testing.com/", ), ) From 5e17799dfb97a84af5b5416733b3df3ba5308ff5 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 11:39:26 +0200 Subject: [PATCH 05/22] fix validation --- .../tests/unit/with_dbs/conftest.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py index 6f240d658ee..b837554cc50 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -19,6 +19,7 @@ RabbitResourceTrackingMessageType, RabbitResourceTrackingStartedMessage, ) +from pydantic import TypeAdapter from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from servicelib.rabbitmq import RabbitMQRPCClient @@ -215,7 +216,9 @@ def random_rabbit_message_heartbeat( def _creator(**kwargs: dict[str, Any]) -> RabbitResourceTrackingHeartbeatMessage: msg_config = {"service_run_id": faker.uuid4(), **kwargs} - return RabbitResourceTrackingHeartbeatMessage(**msg_config) + return TypeAdapter(RabbitResourceTrackingHeartbeatMessage).validate_python( + msg_config + ) return _creator @@ -265,7 +268,9 @@ def _creator(**kwargs: dict[str, Any]) -> RabbitResourceTrackingStartedMessage: **kwargs, } - return RabbitResourceTrackingStartedMessage(**msg_config) + return TypeAdapter(RabbitResourceTrackingStartedMessage).validate_python( + msg_config + ) return _creator From d6133d08f526c56a5c25d1a9324c18a60ff50d4f Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 11:50:41 +0200 Subject: [PATCH 06/22] fix field prop --- .../src/simcore_service_resource_usage_tracker/core/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py index f99fe2cdbed..e3fec5b845d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/settings.py @@ -121,7 +121,7 @@ class ApplicationSettings(MinimalApplicationSettings): json_schema_extra={"auto_default_from_env": True}, ) RESOURCE_USAGE_TRACKER_TRACING: TracingSettings | None = Field( - auto_default_from_env=True, description="settings for opentelemetry tracing" + description="settings for opentelemetry tracing", json_schema_extra={"auto_default_from_env": True} ) @field_validator( From 42cc6397b04d9acab359983e8dc82b1079c7e1cf Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 11:53:58 +0200 Subject: [PATCH 07/22] fix validation --- .../resource_tracker_process_messages.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py index efce3cc4379..7c925244f8d 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/resource_tracker_process_messages.py @@ -47,7 +47,7 @@ async def process_message(app: FastAPI, data: bytes) -> bool: rabbit_message: RabbitResourceTrackingMessages = TypeAdapter( RabbitResourceTrackingMessages - ).validate_python(data) + ).validate_json(data) _logger.info( "Process %s msg service_run_id: %s", rabbit_message.message_type, From 462aa6f207b7ebef49e8ed343b42ed6b6699281e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 11:58:24 +0200 Subject: [PATCH 08/22] fix typecheck --- .../simcore_service_resource_usage_tracker/core/application.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py index 5c1ee907956..e0cef946643 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py @@ -62,7 +62,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: ) # ERROR HANDLERS - app.add_exception_handler(CustomResourceUsageTrackerError, http404_error_handler) + app.add_exception_handler(CustomResourceUsageTrackerError, http404_error_handler) # type: ignore[arg-type] if settings.RESOURCE_USAGE_TRACKER_POSTGRES: setup_db(app) From 729b49515d5e258f3b201dd23338df557feee87d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 13:25:40 +0200 Subject: [PATCH 09/22] fix --- .../services/resource_tracker_pricing_plans.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py index faf7335e1ac..387c8146734 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py @@ -13,6 +13,7 @@ PricingPlanUpdate, ) from models_library.services import ServiceKey, ServiceVersion +from pydantic import TypeAdapter from servicelib.rabbitmq.rpc_interfaces.resource_usage_tracker.errors import ( CustomResourceUsageTrackerError, ) @@ -96,7 +97,7 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ] = await resource_tracker_repo.list_connected_services_to_pricing_plan_by_pricing_plan( product_name=product_name, pricing_plan_id=pricing_plan_id ) - return [PricingPlanToServiceGet.model_validate(item) for item in output_list] + return [TypeAdapter(PricingPlanToServiceGet).validate_python(item) for item in output_list] async def connect_service_to_pricing_plan( @@ -116,7 +117,7 @@ async def connect_service_to_pricing_plan( service_version=service_version, ) ) - return PricingPlanToServiceGet.model_validate(output) + return TypeAdapter(PricingPlanToServiceGet).validate_python(output) async def list_pricing_plans_by_product( From 98a2457cbf0d35534d534028ddc0310917191368 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 14:28:26 +0200 Subject: [PATCH 10/22] fix test --- .../with_dbs/test_api_resource_tracker_credit_transactions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py index 1e7098cecda..d0e24e54661 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py @@ -73,7 +73,7 @@ async def test_credit_transactions_workflow( ) assert response.status_code == status.HTTP_201_CREATED data = response.json() - data["credit_transaction_id"] == 2 + assert data["credit_transaction_id"] == 2 response = await async_client.post( url=f"{url}", @@ -99,4 +99,4 @@ async def test_credit_transactions_workflow( assert response.status_code == status.HTTP_200_OK data = response.json() assert data["wallet_id"] == 1 - assert data["available_osparc_credits"] == Decimal(1340.04) + assert data["available_osparc_credits"] == "1340.04" From 6d663fefc3f58b8048aa4037d258e35b174a0daa Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 14:36:20 +0200 Subject: [PATCH 11/22] fix --- .../services/resource_tracker_pricing_plans.py | 4 ++-- .../with_dbs/test_api_resource_tracker_credit_transactions.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py index 387c8146734..9e191f4c946 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/services/resource_tracker_pricing_plans.py @@ -97,7 +97,7 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ] = await resource_tracker_repo.list_connected_services_to_pricing_plan_by_pricing_plan( product_name=product_name, pricing_plan_id=pricing_plan_id ) - return [TypeAdapter(PricingPlanToServiceGet).validate_python(item) for item in output_list] + return [TypeAdapter(PricingPlanToServiceGet).validate_python(item.model_dump()) for item in output_list] async def connect_service_to_pricing_plan( @@ -117,7 +117,7 @@ async def connect_service_to_pricing_plan( service_version=service_version, ) ) - return TypeAdapter(PricingPlanToServiceGet).validate_python(output) + return TypeAdapter(PricingPlanToServiceGet).validate_python(output.model_dump()) async def list_pricing_plans_by_product( diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py index d0e24e54661..6f4693dbb6e 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py @@ -1,5 +1,4 @@ from collections.abc import Iterator -from decimal import Decimal from typing import Callable import httpx From b3d96923d8773eb61140a5040c1765b9fe659c9d Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 14:44:37 +0200 Subject: [PATCH 12/22] fix --- .../api_schemas_resource_usage_tracker/credit_transactions.py | 2 +- .../with_dbs/test_api_resource_tracker_credit_transactions.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py index c24291ebbdb..272b297df59 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py @@ -13,7 +13,7 @@ class WalletTotalCredits(BaseModel): wallet_id: WalletID available_osparc_credits: Decimal - @field_validator("available_osparc_credits") + @field_validator("available_osparc_credits", mode="before") @classmethod def ensure_rounded(cls, v): return round(v, 2) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py index 6f4693dbb6e..5429228f8b0 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py @@ -1,4 +1,5 @@ from collections.abc import Iterator +from decimal import Decimal from typing import Callable import httpx @@ -98,4 +99,4 @@ async def test_credit_transactions_workflow( assert response.status_code == status.HTTP_200_OK data = response.json() assert data["wallet_id"] == 1 - assert data["available_osparc_credits"] == "1340.04" + assert data["available_osparc_credits"] == Decimal(1340.04) From cd27d9863a53323fcfe41b8468ecbe0b7b4a806b Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Wed, 16 Oct 2024 14:53:43 +0200 Subject: [PATCH 13/22] fix --- .../with_dbs/test_api_resource_tracker_credit_transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py index 5429228f8b0..244a74c62d7 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_api_resource_tracker_credit_transactions.py @@ -56,7 +56,7 @@ async def test_credit_transactions_workflow( ) assert response.status_code == status.HTTP_201_CREATED data = response.json() - data["credit_transaction_id"] == 1 + assert data["credit_transaction_id"] == 1 response = await async_client.post( url=f"{url}", From fa263f6fb7b3edb67704801311625bcbcba9e516 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 09:33:49 +0200 Subject: [PATCH 14/22] remove deprecated --- .../db/repositories/resource_tracker.py | 38 ++++++++++--------- .../tests/unit/with_dbs/conftest.py | 4 +- .../unit/with_dbs/test_background_task.py | 8 ++-- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py index 231c97502fb..0e54a83ea2a 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py @@ -161,7 +161,7 @@ async def update_service_run_last_heartbeat( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) async def update_service_run_stopped_at( self, data: ServiceRunStoppedAtUpdate @@ -191,7 +191,7 @@ async def update_service_run_stopped_at( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) async def get_service_run_by_id( self, service_run_id: ServiceRunId @@ -204,7 +204,7 @@ async def get_service_run_by_id( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) async def list_service_runs_by_product_and_user_and_wallet( self, @@ -309,7 +309,9 @@ async def list_service_runs_by_product_and_user_and_wallet( result = await conn.execute(query) - return [ServiceRunWithCreditsDB.from_orm(row) for row in result.fetchall()] + return [ + ServiceRunWithCreditsDB.model_validate(row) for row in result.fetchall() + ] async def get_osparc_credits_aggregated_by_service( self, @@ -405,7 +407,7 @@ async def get_osparc_credits_aggregated_by_service( return ( cast(int, count_result.scalar()), [ - OsparcCreditsAggregatedByServiceKeyDB.from_orm(row) + OsparcCreditsAggregatedByServiceKeyDB.model_validate(row) for row in list_result.fetchall() ], ) @@ -571,7 +573,7 @@ async def list_service_runs_with_running_status_across_all_products( ) result = await conn.execute(query) - return [ServiceRunForCheckDB.from_orm(row) for row in result.fetchall()] + return [ServiceRunForCheckDB.model_validate(row) for row in result.fetchall()] async def total_service_runs_with_running_status_across_all_products( self, @@ -620,7 +622,7 @@ async def update_service_missed_heartbeat_counter( row = result.first() if row is None: return None - return ServiceRunDB.from_orm(row) + return ServiceRunDB.model_validate(row) ################################# # Credit transactions @@ -847,7 +849,7 @@ def _version(column_or_value): result = await conn.execute(query) return [ - PricingPlansWithServiceDefaultPlanDB.from_orm(row) + PricingPlansWithServiceDefaultPlanDB.model_validate(row) for row in result.fetchall() ] @@ -873,7 +875,7 @@ async def get_pricing_plan( raise CustomResourceUsageTrackerError( msg=f"Pricing plan does not exists: {pricing_plan_id}" ) - return PricingPlansDB.from_orm(row) + return PricingPlansDB.model_validate(row) async def list_pricing_plans_by_product( self, product_name: ProductName @@ -890,7 +892,7 @@ async def list_pricing_plans_by_product( ).where(resource_tracker_pricing_plans.c.product_name == product_name) result = await conn.execute(select_stmt) - return [PricingPlansDB.from_orm(row) for row in result.fetchall()] + return [PricingPlansDB.model_validate(row) for row in result.fetchall()] async def create_pricing_plan(self, data: PricingPlanCreate) -> PricingPlansDB: async with self.db_engine.begin() as conn: @@ -924,7 +926,7 @@ async def create_pricing_plan(self, data: PricingPlanCreate) -> PricingPlansDB: raise CustomResourceUsageTrackerError( msg=f"Pricing plan was not created: {data}" ) - return PricingPlansDB.from_orm(row) + return PricingPlansDB.model_validate(row) async def update_pricing_plan( self, product_name: ProductName, data: PricingPlanUpdate @@ -961,7 +963,7 @@ async def update_pricing_plan( row = result.first() if row is None: return None - return PricingPlansDB.from_orm(row) + return PricingPlansDB.model_validate(row) ################################# # Pricing plan to service @@ -1000,7 +1002,9 @@ async def list_connected_services_to_pricing_plan_by_pricing_plan( ) result = await conn.execute(query) - return [PricingPlanToServiceDB.from_orm(row) for row in result.fetchall()] + return [ + PricingPlanToServiceDB.model_validate(row) for row in result.fetchall() + ] async def upsert_service_to_pricing_plan( self, @@ -1087,7 +1091,7 @@ async def upsert_service_to_pricing_plan( raise CustomResourceUsageTrackerError( msg="Pricing plan to service record was not created" ) - return PricingPlanToServiceDB.from_orm(row) + return PricingPlanToServiceDB.model_validate(row) ################################# # Pricing units @@ -1145,7 +1149,7 @@ async def list_pricing_units_by_pricing_plan( ) result = await conn.execute(query) - return [PricingUnitsDB.from_orm(row) for row in result.fetchall()] + return [PricingUnitsDB.model_validate(row) for row in result.fetchall()] async def get_valid_pricing_unit( self, @@ -1197,7 +1201,7 @@ async def get_valid_pricing_unit( raise CustomResourceUsageTrackerError( msg=f"Pricing plan {pricing_plan_id} and pricing unit {pricing_unit_id} for product {product_name} not found" ) - return PricingUnitsDB.from_orm(row) + return PricingUnitsDB.model_validate(row) async def create_pricing_unit_with_cost( self, data: PricingUnitWithCostCreate, pricing_plan_key: str @@ -1348,4 +1352,4 @@ async def get_pricing_unit_cost_by_id( raise CustomResourceUsageTrackerError( msg=f"Pricing unit cost id {pricing_unit_cost_id} not found in the resource_tracker_pricing_unit_costs table", ) - return PricingUnitCostsDB.from_orm(row) + return PricingUnitCostsDB.model_validate(row) diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py index b837554cc50..28ea92020f8 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/conftest.py @@ -177,7 +177,7 @@ async def assert_service_runs_db_row( ) row = result.first() assert row - service_run_db = ServiceRunDB.from_orm(row) + service_run_db = ServiceRunDB.model_validate(row) if status: assert service_run_db.service_run_status == status return service_run_db @@ -202,7 +202,7 @@ async def assert_credit_transactions_db_row( ) row = result.first() assert row - credit_transaction_db = CreditTransactionDB.from_orm(row) + credit_transaction_db = CreditTransactionDB.model_validate(row) if modified_at: assert credit_transaction_db.modified > modified_at return credit_transaction_db diff --git a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py index 2b719326bc9..3bf73f32f6d 100644 --- a/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py +++ b/services/resource-usage-tracker/tests/unit/with_dbs/test_background_task.py @@ -152,7 +152,7 @@ async def test_process_event_functions( # Check max acceptable missed heartbeats reached before considering them as unhealthy with postgres_db.connect() as con: result = con.execute(sa.select(resource_tracker_service_runs)) - service_run_db = [ServiceRunDB.from_orm(row) for row in result] + service_run_db = [ServiceRunDB.model_validate(row) for row in result] for service_run in service_run_db: if service_run.service_run_id in ( _SERVICE_RUN_ID_OSPARC_10_MIN_OLD, @@ -172,7 +172,7 @@ async def test_process_event_functions( with postgres_db.connect() as con: result = con.execute(sa.select(resource_tracker_service_runs)) - service_run_db = [ServiceRunDB.from_orm(row) for row in result] + service_run_db = [ServiceRunDB.model_validate(row) for row in result] for service_run in service_run_db: if service_run.service_run_id in ( _SERVICE_RUN_ID_OSPARC_10_MIN_OLD, @@ -186,7 +186,9 @@ async def test_process_event_functions( with postgres_db.connect() as con: result = con.execute(sa.select(resource_tracker_credit_transactions)) - credit_transaction_db = [CreditTransactionDB.from_orm(row) for row in result] + credit_transaction_db = [ + CreditTransactionDB.model_validate(row) for row in result + ] for transaction in credit_transaction_db: if transaction.service_run_id in ( _SERVICE_RUN_ID_OSPARC_10_MIN_OLD, From 9bc9c7bfc4ef3b407304fc4572f6665d22d83bea Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 09:49:24 +0200 Subject: [PATCH 15/22] change mode --- .../api_schemas_resource_usage_tracker/credit_transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py index 272b297df59..c5d73b4fa21 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py @@ -13,7 +13,7 @@ class WalletTotalCredits(BaseModel): wallet_id: WalletID available_osparc_credits: Decimal - @field_validator("available_osparc_credits", mode="before") + @field_validator("available_osparc_credits", mode="after") @classmethod def ensure_rounded(cls, v): return round(v, 2) From c2c98d33572d19e3ad4d46acb691b94a95ee392e Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 10:13:44 +0200 Subject: [PATCH 16/22] revert credit validator --- .../api_schemas_resource_usage_tracker/credit_transactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py index c5d73b4fa21..c24291ebbdb 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py @@ -13,7 +13,7 @@ class WalletTotalCredits(BaseModel): wallet_id: WalletID available_osparc_credits: Decimal - @field_validator("available_osparc_credits", mode="after") + @field_validator("available_osparc_credits") @classmethod def ensure_rounded(cls, v): return round(v, 2) From a68cdc42873a04507467ec546dbefe89b5ac70d6 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 10:33:23 +0200 Subject: [PATCH 17/22] fix Decimal serialization --- .../credit_transactions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py index c24291ebbdb..b2f6da36724 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py @@ -1,7 +1,7 @@ from datetime import datetime from decimal import Decimal -from pydantic import BaseModel, field_validator +from pydantic import BaseModel, ConfigDict, field_validator from ..products import ProductName from ..resource_tracker import CreditTransactionId @@ -13,7 +13,9 @@ class WalletTotalCredits(BaseModel): wallet_id: WalletID available_osparc_credits: Decimal - @field_validator("available_osparc_credits") + model_config = ConfigDict(json_encoders={Decimal: float}) + + @field_validator("available_osparc_credits", mode="before") @classmethod def ensure_rounded(cls, v): return round(v, 2) From 956494284427f664a47f0ba2feb1ef13ed60153a Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 10:55:11 +0200 Subject: [PATCH 18/22] replace deprecated --- .../modules/db/repositories/resource_tracker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py index 0e54a83ea2a..4358d560396 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/modules/db/repositories/resource_tracker.py @@ -1213,9 +1213,9 @@ async def create_pricing_unit_with_cost( .values( pricing_plan_id=data.pricing_plan_id, unit_name=data.unit_name, - unit_extra_info=data.unit_extra_info.dict(), + unit_extra_info=data.unit_extra_info.model_dump(), default=data.default, - specific_info=data.specific_info.dict(), + specific_info=data.specific_info.model_dump(), created=sa.func.now(), modified=sa.func.now(), ) @@ -1265,9 +1265,9 @@ async def update_pricing_unit_with_cost( resource_tracker_pricing_units.update() .values( unit_name=data.unit_name, - unit_extra_info=data.unit_extra_info.dict(), + unit_extra_info=data.unit_extra_info.model_dump(), default=data.default, - specific_info=data.specific_info.dict(), + specific_info=data.specific_info.model_dump(), modified=sa.func.now(), ) .where( From 7463f605a5556ec118340bdc7507f6650e4325b1 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 11:04:13 +0200 Subject: [PATCH 19/22] add serializer --- .../credit_transactions.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py index b2f6da36724..2d76bad55bc 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/credit_transactions.py @@ -1,7 +1,8 @@ from datetime import datetime from decimal import Decimal +from typing import Annotated -from pydantic import BaseModel, ConfigDict, field_validator +from pydantic import BaseModel, BeforeValidator, PlainSerializer from ..products import ProductName from ..resource_tracker import CreditTransactionId @@ -11,15 +12,11 @@ class WalletTotalCredits(BaseModel): wallet_id: WalletID - available_osparc_credits: Decimal - - model_config = ConfigDict(json_encoders={Decimal: float}) - - @field_validator("available_osparc_credits", mode="before") - @classmethod - def ensure_rounded(cls, v): - return round(v, 2) - + available_osparc_credits: Annotated[ + Decimal, + BeforeValidator(lambda x: round(x, 2)), + PlainSerializer(float, return_type=float, when_used='json') + ] class CreditTransactionCreateBody(BaseModel): product_name: ProductName From ea7536d8bb2390fa941e5048681b1f44a7e2d8ab Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 11:11:43 +0200 Subject: [PATCH 20/22] fix openapi --- services/resource-usage-tracker/openapi.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/resource-usage-tracker/openapi.json b/services/resource-usage-tracker/openapi.json index c7fe85b4297..dc1e068861e 100644 --- a/services/resource-usage-tracker/openapi.json +++ b/services/resource-usage-tracker/openapi.json @@ -558,7 +558,7 @@ "minimum": 0 }, "available_osparc_credits": { - "type": "string", + "type": "number", "title": "Available Osparc Credits" } }, From d7510915cc688f43afd12916d58022c65512aa72 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 11:32:58 +0200 Subject: [PATCH 21/22] fix Decimal field serialization --- .../api_schemas_resource_usage_tracker/pricing_plans.py | 5 +++-- services/resource-usage-tracker/openapi.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py index bbb5d52f906..14069b2dc35 100644 --- a/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py +++ b/packages/models-library/src/models_library/api_schemas_resource_usage_tracker/pricing_plans.py @@ -1,7 +1,8 @@ from datetime import datetime from decimal import Decimal +from typing import Annotated -from pydantic import BaseModel, ConfigDict +from pydantic import BaseModel, ConfigDict, PlainSerializer from ..resource_tracker import ( HardwareInfo, @@ -18,7 +19,7 @@ class PricingUnitGet(BaseModel): pricing_unit_id: PricingUnitId unit_name: str unit_extra_info: UnitExtraInfo - current_cost_per_unit: Decimal + current_cost_per_unit: Annotated[Decimal, PlainSerializer(float, return_type=float, when_used='json')] current_cost_per_unit_id: PricingUnitCostId default: bool specific_info: HardwareInfo diff --git a/services/resource-usage-tracker/openapi.json b/services/resource-usage-tracker/openapi.json index dc1e068861e..6aa53c7118c 100644 --- a/services/resource-usage-tracker/openapi.json +++ b/services/resource-usage-tracker/openapi.json @@ -459,7 +459,7 @@ "$ref": "#/components/schemas/UnitExtraInfo" }, "current_cost_per_unit": { - "type": "string", + "type": "number", "title": "Current Cost Per Unit" }, "current_cost_per_unit_id": { From 768defdf9037dccce266e79033b6832f51a658b0 Mon Sep 17 00:00:00 2001 From: Giancarlo Romeo Date: Mon, 21 Oct 2024 15:05:17 +0200 Subject: [PATCH 22/22] fix error handler --- .../core/application.py | 2 +- .../src/simcore_service_resource_usage_tracker/core/errors.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py index e0cef946643..5c1ee907956 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/application.py @@ -62,7 +62,7 @@ def create_app(settings: ApplicationSettings) -> FastAPI: ) # ERROR HANDLERS - app.add_exception_handler(CustomResourceUsageTrackerError, http404_error_handler) # type: ignore[arg-type] + app.add_exception_handler(CustomResourceUsageTrackerError, http404_error_handler) if settings.RESOURCE_USAGE_TRACKER_POSTGRES: setup_db(app) diff --git a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py index 298e63aef71..44557db0b02 100644 --- a/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py +++ b/services/resource-usage-tracker/src/simcore_service_resource_usage_tracker/core/errors.py @@ -12,8 +12,10 @@ class ConfigurationError(ResourceUsageTrackerRuntimeError): def http404_error_handler( request: Request, # pylint: disable=unused-argument - error: CustomResourceUsageTrackerError, + error: Exception, ) -> JSONResponse: + assert isinstance(error, CustomResourceUsageTrackerError) # nosec + return JSONResponse( status_code=status.HTTP_404_NOT_FOUND, content={"message": f"{error.msg_template}"},