✨ Link assets folder from service to front-end (#4572)

Co-authored-by: Andrei Neagu <[email protected]>

services/dynamic-sidecar/tests/unit/test_modules_outputs_event_handler.py

@@ -4,7 +4,7 @@
 import asyncio
 from pathlib import Path
 from typing import AsyncIterable
-from unittest.mock import AsyncMock
+from unittest.mock import Mock
 
 import aioprocessing
 import pytest
@@ -46,7 +46,7 @@ async def outputs_manager(
     )
     await outputs_manager.start()
 
-    outputs_manager.set_all_ports_for_upload = AsyncMock()
+    outputs_manager.set_all_ports_for_upload = Mock()
 
     yield outputs_manager
     await outputs_manager.shutdown()

services/dynamic-sidecar/tests/unit/test_modules_outputs_watchdog_extensions.py

@@ -2,7 +2,7 @@
 
 import asyncio
 from pathlib import Path
-from unittest.mock import AsyncMock
+from unittest.mock import Mock
 from uuid import uuid4
 
 import pytest
@@ -34,7 +34,7 @@ async def test_regression_watchdog_blocks_on_handler_error(
     path_to_observe: Path, fail_once: bool
 ):
     raised_error = False
-    event_handler = AsyncMock()
+    event_handler = Mock()
 
     class MockedEventHandler(FileSystemEventHandler):
         def on_any_event(self, event: FileSystemEvent) -> None:
@@ -43,7 +43,8 @@ def on_any_event(self, event: FileSystemEvent) -> None:
             nonlocal raised_error
             if not raised_error and fail_once:
                 raised_error = True
-                raise RuntimeError("raised as expected")
+                msg = "raised as expected"
+                raise RuntimeError(msg)
 
     observer = ExtendedInotifyObserver()
     observer.schedule(
@@ -75,7 +76,8 @@ async def test_safe_file_system_event_handler(
     class MockedEventHandler(SafeFileSystemEventHandler):
         def event_handler(self, _: FileSystemEvent) -> None:
             if user_code_raises_error:
-                raise RuntimeError("error was raised")
+                msg = "error was raised"
+                raise RuntimeError(msg)
 
     mocked_handler = MockedEventHandler()
     mocked_handler.on_any_event(mocked_file_system_event)

services/web/server/src/simcore_service_webserver/projects/_nodes_api.py

@@ -2,10 +2,12 @@
 import logging
 import mimetypes
 import urllib.parse
-from typing import Final
+from pathlib import Path
+from typing import Final, NamedTuple
 
 from aiohttp import web
 from aiohttp.client import ClientError
+from models_library.api_schemas_storage import FileMetaDataGet
 from models_library.projects import ProjectID
 from models_library.projects_nodes import Node, NodeID
 from models_library.projects_nodes_io import SimCoreFileLink
@@ -20,15 +22,21 @@
     parse_obj_as,
     root_validator,
 )
+from servicelib.utils import logged_gather
 
-from .._constants import APP_SETTINGS_KEY, RQT_USERID_KEY
 from ..application_settings import get_settings
-from ..storage.api import get_download_link
+from ..storage.api import get_download_link, get_files_in_node_folder
 from .exceptions import ProjectStartsTooManyDynamicNodesError
 
 _logger = logging.getLogger(__name__)
+
 _NODE_START_INTERVAL_S: Final[datetime.timedelta] = datetime.timedelta(seconds=15)
 
+_SUPPORTED_THUMBNAIL_EXTENSIONS: set[str] = {".png", ".jpeg", ".jpg"}
+_SUPPORTED_PREVIEW_FILE_EXTENSIONS: set[str] = {".gltf", ".png", ".jpeg", ".jpg"}
+
+ASSETS_FOLDER: Final[str] = "assets"
+
 
 def get_service_start_lock_key(user_id: UserID, project_uuid: ProjectID) -> str:
     return f"lock_service_start_limit.{user_id}.{project_uuid}"
@@ -68,6 +76,19 @@ def get_total_project_dynamic_nodes_creation_interval(
 #
 
 
+def _guess_mimetype_from_name(name: str) -> str | None:
+    """Tries to guess the mimetype provided a name
+
+    Arguments:
+        name -- path of a file or the file url
+    """
+    _type, _ = mimetypes.guess_type(name)
+    # NOTE: mimetypes.guess_type works differently in our image, therefore made it nullable if
+    # cannot guess type
+    # SEE https://github.com/ITISFoundation/osparc-simcore/issues/4385
+    return _type
+
+
 class NodeScreenshot(BaseModel):
     thumbnail_url: HttpUrl
     file_url: HttpUrl
@@ -86,24 +107,113 @@ def guess_mimetype_if_undefined(cls, values):
             file_url = values["file_url"]
             assert file_url  # nosec
 
-            _type, _encoding = mimetypes.guess_type(file_url)
-            # NOTE: mimetypes.guess_type works differently in our image, therefore made it nullable if
-            # cannot guess type
-            # SEE https://github.com/ITISFoundation/osparc-simcore/issues/4385
-            values["mimetype"] = _type
+            values["mimetype"] = _guess_mimetype_from_name(file_url)
 
         return values
 
 
-async def fake_screenshots_factory(
-    request: web.Request, node_id: NodeID, node: Node
+def __get_search_key(meta_data: FileMetaDataGet) -> str:
+    return f"{meta_data.file_id}".lower()
+
+
+class _FileWithThumbnail(NamedTuple):
+    file: FileMetaDataGet
+    thumbnail: FileMetaDataGet
+
+
+def _get_files_with_thumbnails(
+    assets_files: list[FileMetaDataGet],
+) -> list[_FileWithThumbnail]:
+    """returns a list of tuples where the second entry is the thumbnails"""
+
+    selected_file_entries: dict[str, FileMetaDataGet] = {
+        __get_search_key(f): f
+        for f in assets_files
+        if not f.file_id.endswith(".hidden_do_not_remove")
+        and Path(f.file_id).suffix.lower() in _SUPPORTED_PREVIEW_FILE_EXTENSIONS
+    }
+
+    with_thumbnail_image: list[_FileWithThumbnail] = []
+
+    for selected_file in set(selected_file_entries.keys()):
+        # search for thumbnail
+        thumbnail: FileMetaDataGet | None = None
+        for extension in _SUPPORTED_THUMBNAIL_EXTENSIONS:
+            thumbnail_search_file_name = f"{selected_file}{extension}"
+            if thumbnail_search_file_name in selected_file_entries:
+                thumbnail = selected_file_entries[thumbnail_search_file_name]
+                break
+        if not thumbnail:
+            continue
+
+        # since there is a thumbnail it can be associated to a file
+        with_thumbnail_image.append(
+            _FileWithThumbnail(
+                file=selected_file_entries[selected_file],
+                thumbnail=thumbnail,
+            )
+        )
+        # remove entries which have been used
+        selected_file_entries.pop(
+            __get_search_key(selected_file_entries[selected_file]), None
+        )
+        selected_file_entries.pop(__get_search_key(thumbnail), None)
+
+    no_thumbnail_image: list[_FileWithThumbnail] = [
+        _FileWithThumbnail(file=x, thumbnail=x) for x in selected_file_entries.values()
+    ]
+
+    return with_thumbnail_image + no_thumbnail_image
+
+
+async def __get_link(
+    app: web.Application, user_id: UserID, file_meta_data: FileMetaDataGet
+) -> tuple[str, HttpUrl]:
+    return __get_search_key(file_meta_data), await get_download_link(
+        app,
+        user_id,
+        parse_obj_as(SimCoreFileLink, {"store": "0", "path": file_meta_data.file_id}),
+    )
+
+
+async def _get_node_screenshots(
+    app: web.Application,
+    user_id: UserID,
+    files_with_thumbnails: list[_FileWithThumbnail],
 ) -> list[NodeScreenshot]:
-    """
-    ONLY for testing purposes
+    """resolves links concurrently before returning all the NodeScreenshots"""
 
-    """
-    assert request.app[APP_SETTINGS_KEY].WEBSERVER_DEV_FEATURES_ENABLED  # nosec
-    screenshots = []
+    search_map: dict[str, FileMetaDataGet] = {}
+
+    for entry in files_with_thumbnails:
+        search_map[__get_search_key(entry.file)] = entry.file
+        search_map[__get_search_key(entry.thumbnail)] = entry.thumbnail
+
+    resolved_links: list[tuple[str, HttpUrl]] = await logged_gather(
+        *[__get_link(app, user_id, x) for x in search_map.values()],
+        max_concurrency=10,
+    )
+
+    mapped_http_url: dict[str, HttpUrl] = dict(resolved_links)
+
+    return [
+        NodeScreenshot(
+            mimetype=_guess_mimetype_from_name(e.file.file_id),
+            file_url=mapped_http_url[__get_search_key(e.file)],
+            thumbnail_url=mapped_http_url[__get_search_key(e.thumbnail)],
+        )
+        for e in files_with_thumbnails
+    ]
+
+
+async def get_node_screenshots(
+    app: web.Application,
+    user_id: UserID,
+    project_id: ProjectID,
+    node_id: NodeID,
+    node: Node,
+) -> list[NodeScreenshot]:
+    screenshots: list[NodeScreenshot] = []
 
     if (
         "file-picker" in node.key
@@ -113,14 +223,13 @@ async def fake_screenshots_factory(
         # Example of file that can be added in file-picker:
         # Example https://github.com/Ybalrid/Ogre_glTF/raw/6a59adf2f04253a3afb9459549803ab297932e8d/Media/Monster.glb
         try:
-            user_id = request[RQT_USERID_KEY]
             text = urllib.parse.quote(node.label)
 
             assert node.outputs is not None  # nosec
 
             filelink = parse_obj_as(SimCoreFileLink, node.outputs["outFile"])
 
-            file_url = await get_download_link(request.app, user_id, filelink)
+            file_url = await get_download_link(app, user_id, filelink)
             screenshots.append(
                 NodeScreenshot(
                     thumbnail_url=f"https://placehold.co/170x120?text={text}",  # type: ignore[arg-type]
@@ -135,33 +244,22 @@ async def fake_screenshots_factory(
             )
 
     elif node.key.startswith("simcore/services/dynamic"):
-        # For dynamic services, just create fake images
-
-        # References:
-        # - https://github.com/Ybalrid/Ogre_glTF
-        # - https://placehold.co/
-        # - https://picsum.photos/
-        #
-        count = int(str(node_id.int)[0])
-        text = urllib.parse.quote(node.label)
-
-        screenshots = [
-            *(
-                NodeScreenshot(
-                    thumbnail_url=f"https://picsum.photos/seed/{node_id.int + n}/170/120",  # type: ignore[arg-type]
-                    file_url=f"https://picsum.photos/seed/{node_id.int + n}/500",  # type: ignore[arg-type]
-                    mimetype="image/jpeg",
-                )
-                for n in range(count)
-            ),
-            *(
-                NodeScreenshot(
-                    thumbnail_url=f"https://placehold.co/170x120?text={text}",  # type: ignore[arg-type]
-                    file_url=f"https://placehold.co/500x500?text={text}",  # type: ignore[arg-type]
-                    mimetype="image/svg+xml",
-                )
-                for n in range(count)
-            ),
-        ]
+        # when dealing with dynamic service scan the assets directory and
+        # pull in all the assets that have been dropped in there
+
+        assets_files: list[FileMetaDataGet] = await get_files_in_node_folder(
+            app=app,
+            user_id=user_id,
+            project_id=project_id,
+            node_id=node_id,
+            folder_name=ASSETS_FOLDER,
+        )
+
+        resolved_screenshots: list[NodeScreenshot] = await _get_node_screenshots(
+            app=app,
+            user_id=user_id,
+            files_with_thumbnails=_get_files_with_thumbnails(assets_files),
+        )
+        screenshots.extend(resolved_screenshots)
 
     return screenshots

services/web/server/src/simcore_service_webserver/projects/_nodes_handlers.py

@@ -39,7 +39,6 @@
 from servicelib.mimetype_constants import MIMETYPE_APPLICATION_JSON
 from simcore_postgres_database.models.users import UserRole
 
-from .._constants import APP_SETTINGS_KEY, MSG_UNDER_DEVELOPMENT
 from .._meta import API_VTAG as VTAG
 from ..catalog import client as catalog_client
 from ..director_v2 import api
@@ -50,7 +49,7 @@
 from ..utils_aiohttp import envelope_json_response
 from . import projects_api
 from ._common_models import ProjectPathParams, RequestContext
-from ._nodes_api import NodeScreenshot, fake_screenshots_factory
+from ._nodes_api import NodeScreenshot, get_node_screenshots
 from .db import ProjectDBAPI
 from .exceptions import (
     NodeNotFoundError,
@@ -512,9 +511,6 @@ async def list_project_nodes_previews(request: web.Request) -> web.Response:
     path_params = parse_request_path_parameters_as(ProjectPathParams, request)
     assert req_ctx  # nosec
 
-    if not request.app[APP_SETTINGS_KEY].WEBSERVER_DEV_FEATURES_ENABLED:
-        raise NotImplementedError(MSG_UNDER_DEVELOPMENT)
-
     nodes_previews: list[_ProjectNodePreview] = []
     project_data = await projects_api.get_project_for_user(
         request.app,
@@ -524,7 +520,13 @@ async def list_project_nodes_previews(request: web.Request) -> web.Response:
     project = Project.parse_obj(project_data)
 
     for node_id, node in project.workbench.items():
-        screenshots = await fake_screenshots_factory(request, NodeID(node_id), node)
+        screenshots = await get_node_screenshots(
+            app=request.app,
+            user_id=req_ctx.user_id,
+            project_id=path_params.project_id,
+            node_id=NodeID(node_id),
+            node=node,
+        )
         if screenshots:
             nodes_previews.append(
                 _ProjectNodePreview(
@@ -549,9 +551,6 @@ async def get_project_node_preview(request: web.Request) -> web.Response:
     path_params = parse_request_path_parameters_as(_NodePathParams, request)
     assert req_ctx  # nosec
 
-    if not request.app[APP_SETTINGS_KEY].WEBSERVER_DEV_FEATURES_ENABLED:
-        raise NotImplementedError(MSG_UNDER_DEVELOPMENT)
-
     project_data = await projects_api.get_project_for_user(
         request.app,
         project_uuid=f"{path_params.project_id}",
@@ -567,14 +566,15 @@ async def get_project_node_preview(request: web.Request) -> web.Response:
             node_uuid=f"{path_params.node_id}",
         )
 
-    # NOTE: keep until is not a dev-feature
-    # raise HTTPNotFound(
-    #     reason=f"Node '{path_params.project_id}/{path_params.node_id}' has no preview"
-    # )
-    #
     node_preview = _ProjectNodePreview(
         project_id=project.uuid,
         node_id=path_params.node_id,
-        screenshots=await fake_screenshots_factory(request, path_params.node_id, node),
+        screenshots=await get_node_screenshots(
+            app=request.app,
+            user_id=req_ctx.user_id,
+            project_id=path_params.project_id,
+            node_id=path_params.node_id,
+            node=node,
+        ),
     )
     return envelope_json_response(node_preview)