feat(cli, service): support for project image (#3623)

docs/reference/core.rst

@@ -65,6 +65,10 @@ Schema classes used to serialize domain models to JSON-LD.
    :members:
    :show-inheritance:
 
+.. automodule:: renku.command.schema.image
+   :members:
+   :show-inheritance:
+
 .. automodule:: renku.command.schema.parameter
    :members:
    :show-inheritance:
@@ -105,10 +109,6 @@ Datasets
    :members:
    :show-inheritance:
 
-.. automodule:: renku.core.dataset.request_model
-   :members:
-   :show-inheritance:
-
 .. automodule:: renku.core.dataset.tag
    :members:
    :show-inheritance:
@@ -237,6 +237,17 @@ Errors that can be raised by ``renku.core``.
    :members:
    :show-inheritance:
 
+Project/Dataset Images
+----------------------
+
+.. automodule:: renku.core.image
+   :members:
+   :show-inheritance:
+
+.. automodule:: renku.domain_model.image
+   :members:
+
+
 Utilities
 ---------
 

renku/command/project.py

@@ -17,14 +17,14 @@
 """Project management."""
 
 from renku.command.command_builder.command import Command
-from renku.core.constant import DATABASE_METADATA_PATH
+from renku.core.constant import PROJECT_METADATA_PATH
 from renku.core.project import edit_project, show_project
 
 
 def edit_project_command():
     """Command for editing project metadata."""
     command = Command().command(edit_project).lock_project().with_database(write=True)
-    return command.require_migration().with_commit(commit_only=DATABASE_METADATA_PATH)
+    return command.require_migration().with_commit(commit_only=PROJECT_METADATA_PATH)
 
 
 def show_project_command():

renku/command/schema/dataset.py

@@ -21,7 +21,8 @@
 from renku.command.schema.annotation import AnnotationSchema
 from renku.command.schema.calamus import DateTimeList, JsonLDSchema, Nested, Uri, fields, oa, prov, renku, schema
 from renku.command.schema.entity import CollectionSchema, EntitySchema
-from renku.domain_model.dataset import Dataset, DatasetFile, DatasetTag, ImageObject, Language, RemoteEntity, Url
+from renku.command.schema.image import ImageObjectSchema
+from renku.domain_model.dataset import Dataset, DatasetFile, DatasetTag, Language, RemoteEntity, Url
 
 
 def dump_dataset_as_jsonld(dataset: Dataset) -> dict:
@@ -104,21 +105,6 @@ class Meta:
     name = fields.String(schema.name)
 
 
-class ImageObjectSchema(JsonLDSchema):
-    """ImageObject schema."""
-
-    class Meta:
-        """Meta class."""
-
-        rdf_type = schema.ImageObject
-        model = ImageObject
-        unknown = EXCLUDE
-
-    content_url = fields.String(schema.contentUrl)
-    id = fields.Id(load_default=None)
-    position = fields.Integer(schema.position)
-
-
 class RemoteEntitySchema(JsonLDSchema):
     """RemoteEntity schema."""
 

renku/command/schema/image.py

@@ -0,0 +1,36 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image JSON-LD schema."""
+
+from marshmallow import EXCLUDE
+
+from renku.command.schema.calamus import JsonLDSchema, fields, schema
+from renku.domain_model.image import ImageObject
+
+
+class ImageObjectSchema(JsonLDSchema):
+    """ImageObject schema."""
+
+    class Meta:
+        """Meta class."""
+
+        rdf_type = schema.ImageObject
+        model = ImageObject
+        unknown = EXCLUDE
+
+    content_url = fields.String(schema.contentUrl)
+    id = fields.Id(load_default=None)
+    position = fields.Integer(schema.position)

renku/command/schema/project.py

@@ -20,6 +20,7 @@
 from renku.command.schema.agent import PersonSchema
 from renku.command.schema.annotation import AnnotationSchema
 from renku.command.schema.calamus import DateTimeList, JsonLDSchema, Nested, StringList, fields, oa, prov, renku, schema
+from renku.command.schema.image import ImageObjectSchema
 from renku.domain_model.project import Project
 
 
@@ -39,6 +40,7 @@ class Meta:
     date_created = DateTimeList(schema.dateCreated, load_default=None, format="iso", extra_formats=("%Y-%m-%d",))
     description = fields.String(schema.description, load_default=None)
     id = fields.Id(load_default=None)
+    image = fields.Nested(schema.image, ImageObjectSchema, load_default=None)
     immutable_template_files = fields.List(
         renku.immutableTemplateFiles,
         fields.String(),

renku/core/constant.py

@@ -19,6 +19,9 @@
 from enum import IntEnum
 from pathlib import Path
 
+FILESYSTEM_ROOT = os.path.abspath(os.sep)
+"""Path to the root of the filesystem."""
+
 APP_NAME = "Renku"
 """Application name for storing configuration."""
 
@@ -41,6 +44,9 @@
 DATASET_IMAGES = "dataset_images"
 """Directory for dataset images."""
 
+IMAGES = "images"
+"""Path for images/icons."""
+
 DEFAULT_DATA_DIR = "data"
 
 DOCKERFILE = "Dockerfile"
@@ -79,6 +85,11 @@
     Path(RENKU_HOME) / DATABASE_PATH,
 ]
 
+PROJECT_METADATA_PATH = [
+    Path(RENKU_HOME) / DATABASE_PATH,
+    Path(RENKU_HOME) / IMAGES,
+]
+
 DATASET_METADATA_PATHS = [
     Path(RENKU_HOME) / DATABASE_PATH,
     Path(RENKU_HOME) / DATASET_IMAGES,

renku/core/dataset/dataset.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 """Dataset business logic."""
 
+import imghdr
 import os
 import shutil
 import urllib
@@ -35,8 +36,8 @@
 from renku.core.dataset.providers.factory import ProviderFactory
 from renku.core.dataset.providers.git import GitProvider
 from renku.core.dataset.providers.models import DatasetUpdateAction, ProviderDataset
-from renku.core.dataset.request_model import ImageRequestModel
 from renku.core.dataset.tag import get_dataset_by_tag, prompt_access_token, prompt_tag_selection
+from renku.core.image import ImageObjectRequest
 from renku.core.interface.dataset_gateway import IDatasetGateway
 from renku.core.storage import check_external_storage, track_paths_in_storage
 from renku.core.util import communication
@@ -50,6 +51,7 @@
     get_absolute_path,
     get_file_size,
     get_files,
+    get_relative_path,
     get_safe_relative_path,
     hash_file,
     is_path_empty,
@@ -109,7 +111,7 @@ def create_dataset(
     description: Optional[str] = None,
     creators: Optional[List[Person]] = None,
     keywords: Optional[List[str]] = None,
-    images: Optional[List[ImageRequestModel]] = None,
+    images: Optional[List[ImageObjectRequest]] = None,
     update_provenance: bool = True,
     custom_metadata: Optional[Dict[str, Any]] = None,
     storage: Optional[str] = None,
@@ -123,7 +125,7 @@ def create_dataset(
         description(Optional[str], optional): Dataset description (Default value = None).
         creators(Optional[List[Person]], optional): Dataset creators (Default value = None).
         keywords(Optional[List[str]], optional): Dataset keywords (Default value = None).
-        images(Optional[List[ImageRequestModel]], optional): Dataset images (Default value = None).
+        images(Optional[List[ImageObjectRequest]], optional): Dataset images (Default value = None).
         update_provenance(bool, optional): Whether to add this dataset to dataset provenance
             (Default value = True).
         custom_metadata(Optional[Dict[str, Any]], optional): Custom JSON-LD metadata (Default value = None).
@@ -199,7 +201,7 @@ def edit_dataset(
     description: Optional[Union[str, NoValueType]],
     creators: Optional[Union[List[Person], NoValueType]],
     keywords: Optional[Union[List[str], NoValueType]] = NO_VALUE,
-    images: Optional[Union[List[ImageRequestModel], NoValueType]] = NO_VALUE,
+    images: Optional[Union[List[ImageObjectRequest], NoValueType]] = NO_VALUE,
     custom_metadata: Optional[Union[Dict, List[Dict], NoValueType]] = NO_VALUE,
     custom_metadata_source: Optional[Union[str, NoValueType]] = NO_VALUE,
 ):
@@ -211,7 +213,7 @@ def edit_dataset(
         description(Optional[Union[str, NoValueType]]): New description for the dataset.
         creators(Optional[Union[List[Person], NoValueType]]): New creators for the dataset.
         keywords(Optional[Union[List[str], NoValueType]]): New keywords for dataset (Default value = ``NO_VALUE``).
-        images(Optional[Union[List[ImageRequestModel], NoValueType]]): New images for dataset
+        images(Optional[Union[List[ImageObjectRequest], NoValueType]]): New images for dataset
             (Default value = ``NO_VALUE``).
         custom_metadata(Optional[Union[Dict, List[Dict], NoValueType]]): Custom JSON-LD metadata
             (Default value = ``NO_VALUE``).
@@ -248,7 +250,7 @@ def edit_dataset(
     if images == NO_VALUE:
         images_updated = False
     else:
-        images_updated = set_dataset_images(dataset=dataset, images=cast(Optional[List[ImageRequestModel]], images))
+        images_updated = set_dataset_images(dataset=dataset, images=cast(Optional[List[ImageObjectRequest]], images))
 
     if images_updated:
         updated["images"] = (
@@ -855,12 +857,12 @@ def add_datadir_files_to_dataset(dataset: Dataset) -> None:
         dataset.add_or_update_files(dataset_files)
 
 
-def set_dataset_images(dataset: Dataset, images: Optional[List[ImageRequestModel]]):
+def set_dataset_images(dataset: Dataset, images: Optional[List[ImageObjectRequest]]):
     """Set a dataset's images.
 
     Args:
         dataset(Dataset): The dataset to set images on.
-        images(List[ImageRequestModel]): The images to set.
+        images(List[ImageObjectRequest]): The images to set.
 
     Returns:
         True if images were set/modified.
@@ -875,10 +877,30 @@ def set_dataset_images(dataset: Dataset, images: Optional[List[ImageRequestModel
     dataset.images = []
     images_updated = False
     for img in images:
-        img_object = img.to_image_object(dataset)
+        image_folder = project_context.dataset_images_path / dataset.initial_identifier
+        try:
+            img_object = img.to_image_object(owner_id=dataset.id)
+        except errors.ImageError as e:
+            raise errors.DatasetImageError(e) from e
+
+        path = img_object.content_url
+
+        if not img_object.is_remote:
+            # NOTE: only copy dataset image if it's not in .renku/datasets/<id>/images/ already
+            if not path.startswith(str(image_folder)):
+                image_type = imghdr.what(path)
+                if image_type:
+                    ext = f".{image_type}"
+                else:
+                    _, ext = os.path.splitext(path)
+                target_image_path: Union[Path, str] = image_folder / f"{img_object.position}{ext}"
 
-        if not img_object:
-            continue
+                image_folder.parent.mkdir(parents=True, exist_ok=True)
+                shutil.copy(path, target_image_path)
+            else:
+                target_image_path = path
+
+            img_object.content_url = get_relative_path(target_image_path, base=project_context.path)  # type: ignore
 
         if any(i.position == img_object.position for i in dataset.images):
             raise errors.DatasetImageError(f"Duplicate dataset image specified for position {img_object.position}")