✨ public API 0.4.5: accepts list as Job input/output arguments 🚨 (#…

…4239)

packages/models-library/src/models_library/utils/pydantic_tools_extension.py

@@ -1,12 +1,12 @@
-from typing import Optional, TypeVar
+from typing import TypeVar
 
 from pydantic import ValidationError
 from pydantic.tools import parse_obj_as
 
 T = TypeVar("T")
 
 
-def parse_obj_or_none(type_: type[T], obj) -> Optional[T]:
+def parse_obj_or_none(type_: type[T], obj) -> T | None:
     try:
         return parse_obj_as(type_, obj)
     except ValidationError:

packages/service-library/src/servicelib/fastapi/openapi.py

@@ -1,6 +1,7 @@
 """ Common utils for core/application openapi specs
 """
 
+import re
 import types
 from typing import Any
 
@@ -60,9 +61,8 @@ def redefine_operation_id_in_router(router: APIRouter, operation_id_prefix: str)
 
 
 # https://swagger.io/docs/specification/data-models/data-types/#numbers
-SCHEMA_TO_PYTHON_TYPES = {"integer": int, "number": float}
-
-SKIP = (
+_SCHEMA_TO_PYTHON_TYPES = {"integer": int, "number": float}
+_SKIP = (
     "examples",
     # SEE openapi-standard: https://swagger.io/docs/specification/adding-examples/
     # - exampleS are Dicts and not Lists
@@ -79,29 +79,52 @@ def patch_openapi_specs(app_openapi: dict[str, Any]):
     Modifies fastapi auto-generated OAS to pass our openapi validation.
     """
 
+    def _remove_named_groups(regex: str) -> str:
+        # Fixes structure error produce by named groups like
+        # ^simcore/services/comp/(?P<subdir>[a-z0-9][a-z0-9_.-]*/)*(?P<name>[a-z0-9-_]+[a-z0-9])$
+        # into
+        # ^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$
+        return re.sub(r"\(\?P<[^>]+>", "(", regex)
+
+    def _patch_node(key, node):
+        # SEE fastapi ISSUE: https://github.com/tiangolo/fastapi/issues/240 (test_openap.py::test_exclusive_min_openapi_issue )
+        # SEE openapi-standard: https://swagger.io/docs/specification/data-models/data-types/#range
+        node_type = node.get("type")
+
+        if key == "exclusiveMinimum":
+            cast_to_python = _SCHEMA_TO_PYTHON_TYPES[node_type]
+            node["minimum"] = cast_to_python(node[key])
+            node["exclusiveMinimum"] = True
+
+        elif key == "exclusiveMaximum":
+            cast_to_python = _SCHEMA_TO_PYTHON_TYPES[node_type]
+            node["maximum"] = cast_to_python(node[key])
+            node["exclusiveMaximum"] = True
+
+        elif key in ("minimum", "maximum"):
+            # NOTE: Security Audit Report:
+            #   The property in question requires a value of the type integer, but the value you have defined does not match this.
+            #   SEE https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.2.md#dataTypeFormat
+            cast_to_python = _SCHEMA_TO_PYTHON_TYPES[node_type]
+            node[key] = cast_to_python(node[key])
+
+        elif key == "pattern" and node_type == "string":
+            node[key] = _remove_named_groups(regex=node[key])
+
     def _patch(node):
         if isinstance(node, dict):
             for key in list(node.keys()):
-                # SEE fastapi ISSUE: https://github.com/tiangolo/fastapi/issues/240 (test_openap.py::test_exclusive_min_openapi_issue )
-                # SEE openapi-standard: https://swagger.io/docs/specification/data-models/data-types/#range
-                if key == "exclusiveMinimum":
-                    cast_to_python = SCHEMA_TO_PYTHON_TYPES[node["type"]]
-                    node["minimum"] = cast_to_python(node[key])
-                    node["exclusiveMinimum"] = True
-
-                elif key == "exclusiveMaximum":
-                    cast_to_python = SCHEMA_TO_PYTHON_TYPES[node["type"]]
-                    node["maximum"] = cast_to_python(node[key])
-                    node["exclusiveMaximum"] = True
-
-                elif key in SKIP:
+                if key in _SKIP:
                     node.pop(key)
                     continue
+                _patch_node(key, node)
 
+                # recursive
                 _patch(node[key])
 
         elif isinstance(node, list):
             for value in node:
+                # recursive
                 _patch(value)
 
     _patch(app_openapi)

services/api-server/VERSION

@@ -1 +1 @@
-0.4.4
+0.4.5

services/api-server/openapi.json

@@ -377,7 +377,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -426,7 +426,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -479,7 +479,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -538,7 +538,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -599,7 +599,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -668,7 +668,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -737,7 +737,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -767,7 +767,7 @@
             "required": false,
             "schema": {
               "title": "Cluster Id",
-              "minimum": 0.0,
+              "minimum": 0,
               "type": "integer"
             },
             "name": "cluster_id",
@@ -815,7 +815,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -883,7 +883,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -951,7 +951,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -1020,7 +1020,7 @@
             "required": true,
             "schema": {
               "title": "Solver Key",
-              "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+              "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
               "type": "string"
             },
             "name": "solver_key",
@@ -1201,7 +1201,7 @@
           },
           "name": {
             "title": "Name",
-            "pattern": "^([^\\s/]+/?)+$",
+            "pattern": "^([^\\s/]+/?){1,10}$",
             "type": "string"
           },
           "inputs_checksum": {
@@ -1217,7 +1217,7 @@
           },
           "runner_name": {
             "title": "Runner Name",
-            "pattern": "^([^\\s/]+/?)+$",
+            "pattern": "^([^\\s/]+/?){1,10}$",
             "type": "string",
             "description": "Runner that executes job"
           },
@@ -1283,6 +1283,10 @@
                 },
                 {
                   "type": "string"
+                },
+                {
+                  "type": "array",
+                  "items": {}
                 }
               ]
             }
@@ -1295,9 +1299,8 @@
             "title": "Temperature",
             "enabled": true,
             "input_file": {
-              "id": "0a3b2c56-dbcd-4871-b93b-d454b7883f9f",
               "filename": "input.txt",
-              "content_type": "text/plain"
+              "id": "0a3b2c56-dbcd-4871-b93b-d454b7883f9f"
             }
           }
         }
@@ -1335,6 +1338,10 @@
                 },
                 {
                   "type": "string"
+                },
+                {
+                  "type": "array",
+                  "items": {}
                 }
               ]
             }
@@ -1348,9 +1355,8 @@
             "title": "Specific Absorption Rate",
             "enabled": false,
             "output_file": {
-              "id": "0a3b2c56-dbcd-4871-b93b-d454b7883f9f",
               "filename": "sar_matrix.txt",
-              "content_type": "text/plain"
+              "id": "0a3b2c56-dbcd-4871-b93b-d454b7883f9f"
             }
           }
         }
@@ -1374,8 +1380,8 @@
           },
           "progress": {
             "title": "Progress",
-            "maximum": 100.0,
-            "minimum": 0.0,
+            "maximum": 100,
+            "minimum": 0,
             "type": "integer",
             "default": 0
           },
@@ -1409,7 +1415,9 @@
         "title": "Meta",
         "required": [
           "name",
-          "version"
+          "version",
+          "docs_url",
+          "docs_dev_url"
         ],
         "type": "object",
         "properties": {
@@ -1436,16 +1444,14 @@
             "maxLength": 65536,
             "minLength": 1,
             "type": "string",
-            "format": "uri",
-            "default": "https://docs.osparc.io"
+            "format": "uri"
           },
           "docs_dev_url": {
             "title": "Docs Dev Url",
             "maxLength": 65536,
             "minLength": 1,
             "type": "string",
-            "format": "uri",
-            "default": "https://api.osparc.io/dev/docs"
+            "format": "uri"
           }
         },
         "example": {
@@ -1455,8 +1461,8 @@
             "v1": "1.3.4",
             "v2": "2.4.45"
           },
-          "doc_url": "https://api.osparc.io/doc",
-          "doc_dev_url": "https://api.osparc.io/dev/doc"
+          "docs_url": "https://api.osparc.io/doc",
+          "docs_dev_url": "https://api.osparc.io/dev/doc"
         }
       },
       "Profile": {
@@ -1545,7 +1551,7 @@
         "properties": {
           "id": {
             "title": "Id",
-            "pattern": "^(simcore)/(services)/comp(/[\\w/-]+)+$",
+            "pattern": "^simcore/services/comp/([a-z0-9][a-z0-9_.-]*/)*([a-z0-9-_]+[a-z0-9])$",
             "type": "string",
             "description": "Solver identifier"
           },

services/api-server/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.4
+current_version = 0.4.5
 commit = True
 message = services/api-server version: {current_version} → {new_version}
 tag = False

services/api-server/src/simcore_service_api_server/api/routes/solvers_jobs.py

@@ -16,7 +16,7 @@
 from ...models.basic_types import VersionStr
 from ...models.domain.projects import NewProjectIn, Project
 from ...models.schemas.files import File
-from ...models.schemas.jobs import ArgumentType, Job, JobInputs, JobOutputs, JobStatus
+from ...models.schemas.jobs import ArgumentTypes, Job, JobInputs, JobOutputs, JobStatus
 from ...models.schemas.solvers import Solver, SolverKeyId
 from ...plugins.catalog import CatalogApi
 from ...plugins.director_v2 import DirectorV2Api, DownloadLink, NodeName
@@ -26,7 +26,7 @@
     create_jobstatus_from_task,
     create_new_project_for_job,
 )
-from ...utils.solver_job_outputs import get_solver_output_results
+from ...utils.solver_job_outputs import ResultsTypes, get_solver_output_results
 from ..dependencies.application import get_product_name, get_reverse_url_mapper
 from ..dependencies.authentication import get_current_user_id
 from ..dependencies.database import Engine, get_db_engine
@@ -253,16 +253,14 @@ async def get_job_outputs(
     node_ids = list(project.workbench.keys())
     assert len(node_ids) == 1  # nosec
 
-    outputs: dict[
-        str, float | int | bool | BaseFileLink | str | None
-    ] = await get_solver_output_results(
+    outputs: dict[str, ResultsTypes] = await get_solver_output_results(
         user_id=user_id,
         project_uuid=job_id,
         node_uuid=UUID(node_ids[0]),
         db_engine=db_engine,
     )
 
-    results: dict[str, ArgumentType] = {}
+    results: dict[str, ArgumentTypes] = {}
     for name, value in outputs.items():
         if isinstance(value, BaseFileLink):
             # TODO: value.path exists??