🔨♻️ Fixes mypy issue after upgrade

.github/workflows/ci-testing-deploy.yml

@@ -84,6 +84,8 @@ jobs:
               - 'packages/dask-task-models-library/**'
               - 'packages/pytest-simcore/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             models-library:
               - 'packages/models-library/**'
               - 'packages/postgres-database/**'
@@ -93,6 +95,8 @@ jobs:
               - 'packages/postgres-database/**'
               - 'packages/pytest-simcore/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             service-integration:
               - 'packages/models-library/**'
               - 'packages/pytest-simcore/**'
@@ -113,28 +117,40 @@ jobs:
               - 'packages/**'
               - 'services/agent/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             api:
               - 'api/**'
             api-server:
               - 'packages/**'
               - 'services/api-server/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             autoscaling:
               - 'packages/**'
               - 'services/autoscaling/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             catalog:
               - 'packages/**'
               - 'services/catalog/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             dask-sidecar:
               - 'packages/**'
               - 'services/dask-sidecar/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             datcore-adapter:
               - 'packages/**'
               - 'services/datcore-adapter/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             director:
               - 'packages/**'
               - 'services/director/**'
@@ -147,10 +163,14 @@ jobs:
               - 'packages/**'
               - 'services/dynamic-sidecar/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             invitations:
               - 'packages/**'
               - 'services/invitations/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             migration:
               - 'packages/**'
               - 'services/migration/**'
@@ -159,13 +179,17 @@ jobs:
               - 'packages/**'
               - 'services/osparc-gateway-server/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             static-webserver:
               - 'services/static-webserver/**'
               - 'services/docker-compose*'
             storage:
               - 'packages/**'
               - 'services/storage/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             webserver:
               - 'packages/**'
               - 'services/web/**'

mypy.ini

@@ -25,9 +25,10 @@ disallow_untyped_defs = False
 # removes all the missing imports stuff from external libraries which is annoying to the least
 ignore_missing_imports = True
 
-plugins = pydantic.mypy, sqlmypy
+plugins = pydantic.mypy, sqlalchemy.ext.mypy.plugin
 
 [pydantic-mypy]
+# SEE https://docs.pydantic.dev/mypy_plugin/#plugin-settings
 init_forbid_extra = True
 init_typed = True
 warn_required_dynamic_aliases = True

packages/postgres-database/src/simcore_postgres_database/migration/versions/53e095260441_add_projects_access_rights_and_.py

@@ -7,6 +7,7 @@
 """
 import sqlalchemy as sa
 from alembic import op
+from sqlalchemy.dialects.postgresql import JSONB
 
 # revision identifiers, used by Alembic.
 revision = "53e095260441"
@@ -21,7 +22,7 @@ def upgrade():
         "projects",
         sa.Column(
             "access_rights",
-            sa.dialects.postgresql.JSONB(),
+            JSONB,
             nullable=False,
             server_default=sa.text("'{}'::jsonb"),
         ),

packages/postgres-database/src/simcore_postgres_database/migration/versions/da8abd0d8e42_add_comp_runs_table.py

@@ -7,6 +7,7 @@
 """
 import sqlalchemy as sa
 from alembic import op
+from sqlalchemy.dialects.postgresql import ENUM
 
 # revision identifiers, used by Alembic.
 revision = "da8abd0d8e42"
@@ -31,7 +32,7 @@ def upgrade():
         sa.Column("iteration", sa.BigInteger(), autoincrement=False, nullable=False),
         sa.Column(
             "result",
-            sa.dialects.postgresql.ENUM(
+            ENUM(
                 "NOT_STARTED",
                 "PUBLISHED",
                 "PENDING",

packages/postgres-database/src/simcore_postgres_database/utils_aiopg_orm.py

@@ -10,7 +10,7 @@
 
 import functools
 import operator
-from typing import Generic, Optional, TypeVar, Union
+from typing import Any, Generic, TypeVar
 
 import sqlalchemy as sa
 from aiopg.sa.connection import SAConnection
@@ -25,7 +25,7 @@
 RowUId = TypeVar("RowUId", int, str)  # typically id or uuid
 
 
-def _normalize(names: Union[str, list[str], None]) -> list[str]:
+def _normalize(names: str | list[str] | None) -> list[str]:
     if not names:
         return []
     if isinstance(names, str):
@@ -37,15 +37,17 @@ def _normalize(names: Union[str, list[str], None]) -> list[str]:
 ALL_COLUMNS = f"{__name__}.ALL_COLUMNS"
 PRIMARY_KEY = f"{__name__}.PRIMARY_KEY"
 
+QueryT = TypeVar("QueryT", bound=UpdateBase)
+
 
 class BaseOrm(Generic[RowUId]):
     def __init__(
         self,
         table: sa.Table,
         connection: SAConnection,
         *,
-        readonly: Optional[set] = None,
-        writeonce: Optional[set] = None,
+        readonly: set | None = None,
+        writeonce: set | None = None,
     ):
         """
         :param readonly: read-only columns typically created in the server side, defaults to None
@@ -57,7 +59,7 @@ def __init__(
         self._writeonce: set = writeonce or set()
 
         # row selection logic
-        self._where_clause = None
+        self._where_clause: Any = None
         try:
             self._primary_key: Column = next(c for c in table.columns if c.primary_key)
             # FIXME: how can I compare a concrete with a generic type??
@@ -69,7 +71,7 @@ def __init__(
 
     def _compose_select_query(
         self,
-        columns: Union[str, list[str]],
+        columns: str | list[str],
     ) -> Select:
         column_names: list[str] = _normalize(columns)
 
@@ -87,8 +89,8 @@ def _compose_select_query(
         return query
 
     def _append_returning(
-        self, columns: Union[str, list[str]], query: UpdateBase
-    ) -> tuple[UpdateBase, bool]:
+        self, columns: str | list[str], query: QueryT
+    ) -> tuple[QueryT, bool]:
         column_names: list[str] = _normalize(columns)
 
         is_scalar: bool = len(column_names) == 1
@@ -117,14 +119,14 @@ def _check_access_rights(access: set, values: dict) -> None:
     def columns(self) -> ImmutableColumnCollection:
         return self._table.columns
 
-    def set_filter(self, rowid: Optional[RowUId] = None, **unique_id) -> "BaseOrm":
+    def set_filter(self, rowid: RowUId | None = None, **unique_id) -> "BaseOrm":
         """
         Sets default for read operations either by passing a row identifier or a filter
         """
         if unique_id and rowid:
             raise ValueError("Either identifier or unique condition but not both")
 
-        if rowid:
+        if rowid is not None:
             self._where_clause = self._primary_key == rowid
         elif unique_id:
             self._where_clause = functools.reduce(
@@ -149,10 +151,10 @@ def is_filter_set(self) -> bool:
 
     async def fetch(
         self,
-        returning_cols: Union[str, list[str]] = ALL_COLUMNS,
+        returning_cols: str | list[str] = ALL_COLUMNS,
         *,
-        rowid: Optional[RowUId] = None,
-    ) -> Optional[RowProxy]:
+        rowid: RowUId | None = None,
+    ) -> RowProxy | None:
         query = self._compose_select_query(returning_cols)
         if rowid:
             # overrides pinned row
@@ -162,14 +164,13 @@ async def fetch(
             query = query.where(self._where_clause)
 
         result: ResultProxy = await self._conn.execute(query)
-        row: Optional[RowProxy] = await result.first()
+        row: RowProxy | None = await result.first()
         return row
 
     async def fetch_all(
         self,
-        returning_cols: Union[str, list[str]] = ALL_COLUMNS,
+        returning_cols: str | list[str] = ALL_COLUMNS,
     ) -> list[RowProxy]:
-
         query = self._compose_select_query(returning_cols)
         if self.is_filter_set():
             assert self._where_clause is not None  # nosec
@@ -181,10 +182,10 @@ async def fetch_all(
 
     async def fetch_page(
         self,
-        returning_cols: Union[str, list[str]] = ALL_COLUMNS,
+        returning_cols: str | list[str] = ALL_COLUMNS,
         *,
         offset: int,
-        limit: Optional[int] = None,
+        limit: int | None = None,
         sort_by=None,
     ) -> tuple[list[RowProxy], int]:
         """Support for paginated fetchall
@@ -231,8 +232,8 @@ async def fetch_page(
         return rows, total_count
 
     async def update(
-        self, returning_cols: Union[str, list[str]] = PRIMARY_KEY, **values
-    ) -> Union[RowUId, RowProxy, None]:
+        self, returning_cols: str | list[str] = PRIMARY_KEY, **values
+    ) -> RowUId | RowProxy | None:
         self._check_access_rights(self._readonly, values)
         self._check_access_rights(self._writeonce, values)
 
@@ -246,12 +247,12 @@ async def update(
             return await self._conn.scalar(query)
 
         result: ResultProxy = await self._conn.execute(query)
-        row: Optional[RowProxy] = await result.first()
+        row: RowProxy | None = await result.first()
         return row
 
     async def insert(
-        self, returning_cols: Union[str, list[str]] = PRIMARY_KEY, **values
-    ) -> Union[RowUId, RowProxy, None]:
+        self, returning_cols: str | list[str] = PRIMARY_KEY, **values
+    ) -> RowUId | RowProxy | None:
         self._check_access_rights(self._readonly, values)
 
         query: Insert = self._table.insert().values(**values)
@@ -261,5 +262,5 @@ async def insert(
             return await self._conn.scalar(query)
 
         result: ResultProxy = await self._conn.execute(query)
-        row: Optional[RowProxy] = await result.first()
+        row: RowProxy | None = await result.first()
         return row

packages/postgres-database/src/simcore_postgres_database/utils_aiosqlalchemy.py

@@ -1,14 +1,14 @@
-from typing import Any
-
 import sqlalchemy as sa
 from sqlalchemy.ext.asyncio import AsyncEngine
 
 from .utils_migration import get_current_head
 
 
-async def get_pg_engine_stateinfo(engine: AsyncEngine) -> dict[str, Any]:
+async def get_pg_engine_stateinfo(engine: AsyncEngine) -> dict[str, str]:
+    checkedin = engine.pool.checkedin()  # type: ignore
+    checkedout = engine.pool.checkedout()  # type: ignore
     return {
-        "current pool connections": f"{engine.pool.checkedin()=},{engine.pool.checkedout()=}",
+        "current pool connections": f"{checkedin=},{checkedout=}",
     }
 
 

scripts/mypy.bash

@@ -22,6 +22,19 @@ build() {
     "$SCRIPT_DIR/mypy"
 }
 
+echo_requirements() {
+  echo "Installed :"
+  docker run \
+    --interactive \
+    --rm \
+    --user="$(id --user "$USER")":"$(id --group "$USER")" \
+    --entrypoint="pip" \
+    "$IMAGE_NAME" \
+     --no-cache-dir freeze
+}
+
+
+
 run() {
   echo Using "$(docker run --rm "$IMAGE_NAME" --version)"
   echo Mypy config "${MYPY_CONFIG}"
@@ -45,6 +58,7 @@ run() {
 # USAGE
 #    ./scripts/mypy.bash --help
 build
+echo_requirements
 run "$@"
 echo "DONE"
 # ----------------------------------------------------------------------

scripts/mypy/Dockerfile

@@ -4,7 +4,8 @@ FROM python:${PYTHON_VERSION}-slim-buster as base
 
 COPY requirements.txt /requirements.txt
 
-RUN pip install --upgrade pip \
-  && pip install -r requirements.txt
+RUN pip --no-cache-dir install --upgrade pip \
+  && pip --no-cache-dir install -r requirements.txt \
+  && pip --no-cache-dir freeze
 
 ENTRYPOINT ["mypy", "--config-file", "/config/mypy.ini", "--warn-unused-configs"]

scripts/mypy/requirements.txt

@@ -1,6 +1,6 @@
 mypy~=1.2.0
-pydantic[email]
-sqlalchemy-stubs
+pydantic[email]~=1.10  # plugin for primary lib: keep in sync. SEE https://docs.pydantic.dev/mypy_plugin/#plugin-settings
+sqlalchemy[mypy]~=1.4  # plugin for primary lib: keep in sync. SEE https://docs.sqlalchemy.org/en/14/orm/extensions/mypy.html#installation
 types-aiofiles
 types-attrs
 types-PyYAML

services/api-server/src/simcore_service_api_server/models/schemas/jobs.py

@@ -232,7 +232,7 @@ class JobStatus(BaseModel):
 
     job_id: UUID
     state: TaskStates
-    progress: PercentageInt = Field(default=0)
+    progress: PercentageInt = Field(default=PercentageInt(0))
 
     # Timestamps on states
     # TODO: sync state events and timestamps

services/api-server/src/simcore_service_api_server/models/schemas/meta.py

@@ -9,16 +9,16 @@ class Meta(BaseModel):
     released: dict[str, VersionStr] | None = Field(
         None, description="Maps every route's path tag with a released version"
     )
-    docs_url: AnyHttpUrl = Field(default="https://docs.osparc.io")
-    docs_dev_url: AnyHttpUrl = Field(default="https://api.osparc.io/dev/docs")
+    docs_url: AnyHttpUrl
+    docs_dev_url: AnyHttpUrl
 
     class Config:
         schema_extra = {
             "example": {
                 "name": "simcore_service_foo",
                 "version": "2.4.45",
                 "released": {"v1": "1.3.4", "v2": "2.4.45"},
-                "doc_url": "https://api.osparc.io/doc",
-                "doc_dev_url": "https://api.osparc.io/dev/doc",
+                "docs_url": "https://api.osparc.io/doc",
+                "docs_dev_url": "https://api.osparc.io/dev/doc",
             }
         }