diff --git a/docs/reference/core.rst b/docs/reference/core.rst index 19aba89606..2da385b222 100644 --- a/docs/reference/core.rst +++ b/docs/reference/core.rst @@ -65,6 +65,10 @@ Schema classes used to serialize domain models to JSON-LD. :members: :show-inheritance: +.. automodule:: renku.command.schema.image + :members: + :show-inheritance: + .. automodule:: renku.command.schema.parameter :members: :show-inheritance: @@ -105,10 +109,6 @@ Datasets :members: :show-inheritance: -.. automodule:: renku.core.dataset.request_model - :members: - :show-inheritance: - .. automodule:: renku.core.dataset.tag :members: :show-inheritance: @@ -237,6 +237,17 @@ Errors that can be raised by ``renku.core``. :members: :show-inheritance: +Project/Dataset Images +---------------------- + +.. automodule:: renku.core.image + :members: + :show-inheritance: + +.. automodule:: renku.domain_model.image + :members: + + Utilities --------- diff --git a/renku/command/project.py b/renku/command/project.py index d38c9bbb30..82ee538e20 100644 --- a/renku/command/project.py +++ b/renku/command/project.py @@ -17,14 +17,14 @@ """Project management.""" from renku.command.command_builder.command import Command -from renku.core.constant import DATABASE_METADATA_PATH +from renku.core.constant import PROJECT_METADATA_PATH from renku.core.project import edit_project, show_project def edit_project_command(): """Command for editing project metadata.""" command = Command().command(edit_project).lock_project().with_database(write=True) - return command.require_migration().with_commit(commit_only=DATABASE_METADATA_PATH) + return command.require_migration().with_commit(commit_only=PROJECT_METADATA_PATH) def show_project_command(): diff --git a/renku/command/schema/dataset.py b/renku/command/schema/dataset.py index 6987f58b14..e70811d453 100644 --- a/renku/command/schema/dataset.py +++ b/renku/command/schema/dataset.py @@ -21,7 +21,8 @@ from renku.command.schema.annotation import AnnotationSchema from renku.command.schema.calamus import DateTimeList, JsonLDSchema, Nested, Uri, fields, oa, prov, renku, schema from renku.command.schema.entity import CollectionSchema, EntitySchema -from renku.domain_model.dataset import Dataset, DatasetFile, DatasetTag, ImageObject, Language, RemoteEntity, Url +from renku.command.schema.image import ImageObjectSchema +from renku.domain_model.dataset import Dataset, DatasetFile, DatasetTag, Language, RemoteEntity, Url def dump_dataset_as_jsonld(dataset: Dataset) -> dict: @@ -104,21 +105,6 @@ class Meta: name = fields.String(schema.name) -class ImageObjectSchema(JsonLDSchema): - """ImageObject schema.""" - - class Meta: - """Meta class.""" - - rdf_type = schema.ImageObject - model = ImageObject - unknown = EXCLUDE - - content_url = fields.String(schema.contentUrl) - id = fields.Id(load_default=None) - position = fields.Integer(schema.position) - - class RemoteEntitySchema(JsonLDSchema): """RemoteEntity schema.""" diff --git a/renku/command/schema/image.py b/renku/command/schema/image.py new file mode 100644 index 0000000000..a8f38d9a34 --- /dev/null +++ b/renku/command/schema/image.py @@ -0,0 +1,36 @@ +# Copyright Swiss Data Science Center (SDSC). A partnership between +# École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Image JSON-LD schema.""" + +from marshmallow import EXCLUDE + +from renku.command.schema.calamus import JsonLDSchema, fields, schema +from renku.domain_model.image import ImageObject + + +class ImageObjectSchema(JsonLDSchema): + """ImageObject schema.""" + + class Meta: + """Meta class.""" + + rdf_type = schema.ImageObject + model = ImageObject + unknown = EXCLUDE + + content_url = fields.String(schema.contentUrl) + id = fields.Id(load_default=None) + position = fields.Integer(schema.position) diff --git a/renku/command/schema/project.py b/renku/command/schema/project.py index e7833e8a93..4d34cf9838 100644 --- a/renku/command/schema/project.py +++ b/renku/command/schema/project.py @@ -20,6 +20,7 @@ from renku.command.schema.agent import PersonSchema from renku.command.schema.annotation import AnnotationSchema from renku.command.schema.calamus import DateTimeList, JsonLDSchema, Nested, StringList, fields, oa, prov, renku, schema +from renku.command.schema.image import ImageObjectSchema from renku.domain_model.project import Project @@ -39,6 +40,7 @@ class Meta: date_created = DateTimeList(schema.dateCreated, load_default=None, format="iso", extra_formats=("%Y-%m-%d",)) description = fields.String(schema.description, load_default=None) id = fields.Id(load_default=None) + image = fields.Nested(schema.image, ImageObjectSchema, load_default=None) immutable_template_files = fields.List( renku.immutableTemplateFiles, fields.String(), diff --git a/renku/core/constant.py b/renku/core/constant.py index bde8581c07..3cd4b634bc 100644 --- a/renku/core/constant.py +++ b/renku/core/constant.py @@ -19,6 +19,9 @@ from enum import IntEnum from pathlib import Path +FILESYSTEM_ROOT = os.path.abspath(os.sep) +"""Path to the root of the filesystem.""" + APP_NAME = "Renku" """Application name for storing configuration.""" @@ -41,6 +44,9 @@ DATASET_IMAGES = "dataset_images" """Directory for dataset images.""" +IMAGES = "images" +"""Path for images/icons.""" + DEFAULT_DATA_DIR = "data" DOCKERFILE = "Dockerfile" @@ -79,6 +85,11 @@ Path(RENKU_HOME) / DATABASE_PATH, ] +PROJECT_METADATA_PATH = [ + Path(RENKU_HOME) / DATABASE_PATH, + Path(RENKU_HOME) / IMAGES, +] + DATASET_METADATA_PATHS = [ Path(RENKU_HOME) / DATABASE_PATH, Path(RENKU_HOME) / DATASET_IMAGES, diff --git a/renku/core/dataset/dataset.py b/renku/core/dataset/dataset.py index c8456ee8e8..1115db9792 100644 --- a/renku/core/dataset/dataset.py +++ b/renku/core/dataset/dataset.py @@ -15,6 +15,7 @@ # limitations under the License. """Dataset business logic.""" +import imghdr import os import shutil import urllib @@ -35,8 +36,8 @@ from renku.core.dataset.providers.factory import ProviderFactory from renku.core.dataset.providers.git import GitProvider from renku.core.dataset.providers.models import DatasetUpdateAction, ProviderDataset -from renku.core.dataset.request_model import ImageRequestModel from renku.core.dataset.tag import get_dataset_by_tag, prompt_access_token, prompt_tag_selection +from renku.core.image import ImageObjectRequest from renku.core.interface.dataset_gateway import IDatasetGateway from renku.core.storage import check_external_storage, track_paths_in_storage from renku.core.util import communication @@ -50,6 +51,7 @@ get_absolute_path, get_file_size, get_files, + get_relative_path, get_safe_relative_path, hash_file, is_path_empty, @@ -109,7 +111,7 @@ def create_dataset( description: Optional[str] = None, creators: Optional[List[Person]] = None, keywords: Optional[List[str]] = None, - images: Optional[List[ImageRequestModel]] = None, + images: Optional[List[ImageObjectRequest]] = None, update_provenance: bool = True, custom_metadata: Optional[Dict[str, Any]] = None, storage: Optional[str] = None, @@ -123,7 +125,7 @@ def create_dataset( description(Optional[str], optional): Dataset description (Default value = None). creators(Optional[List[Person]], optional): Dataset creators (Default value = None). keywords(Optional[List[str]], optional): Dataset keywords (Default value = None). - images(Optional[List[ImageRequestModel]], optional): Dataset images (Default value = None). + images(Optional[List[ImageObjectRequest]], optional): Dataset images (Default value = None). update_provenance(bool, optional): Whether to add this dataset to dataset provenance (Default value = True). custom_metadata(Optional[Dict[str, Any]], optional): Custom JSON-LD metadata (Default value = None). @@ -199,7 +201,7 @@ def edit_dataset( description: Optional[Union[str, NoValueType]], creators: Optional[Union[List[Person], NoValueType]], keywords: Optional[Union[List[str], NoValueType]] = NO_VALUE, - images: Optional[Union[List[ImageRequestModel], NoValueType]] = NO_VALUE, + images: Optional[Union[List[ImageObjectRequest], NoValueType]] = NO_VALUE, custom_metadata: Optional[Union[Dict, List[Dict], NoValueType]] = NO_VALUE, custom_metadata_source: Optional[Union[str, NoValueType]] = NO_VALUE, ): @@ -211,7 +213,7 @@ def edit_dataset( description(Optional[Union[str, NoValueType]]): New description for the dataset. creators(Optional[Union[List[Person], NoValueType]]): New creators for the dataset. keywords(Optional[Union[List[str], NoValueType]]): New keywords for dataset (Default value = ``NO_VALUE``). - images(Optional[Union[List[ImageRequestModel], NoValueType]]): New images for dataset + images(Optional[Union[List[ImageObjectRequest], NoValueType]]): New images for dataset (Default value = ``NO_VALUE``). custom_metadata(Optional[Union[Dict, List[Dict], NoValueType]]): Custom JSON-LD metadata (Default value = ``NO_VALUE``). @@ -248,7 +250,7 @@ def edit_dataset( if images == NO_VALUE: images_updated = False else: - images_updated = set_dataset_images(dataset=dataset, images=cast(Optional[List[ImageRequestModel]], images)) + images_updated = set_dataset_images(dataset=dataset, images=cast(Optional[List[ImageObjectRequest]], images)) if images_updated: updated["images"] = ( @@ -855,12 +857,12 @@ def add_datadir_files_to_dataset(dataset: Dataset) -> None: dataset.add_or_update_files(dataset_files) -def set_dataset_images(dataset: Dataset, images: Optional[List[ImageRequestModel]]): +def set_dataset_images(dataset: Dataset, images: Optional[List[ImageObjectRequest]]): """Set a dataset's images. Args: dataset(Dataset): The dataset to set images on. - images(List[ImageRequestModel]): The images to set. + images(List[ImageObjectRequest]): The images to set. Returns: True if images were set/modified. @@ -875,10 +877,30 @@ def set_dataset_images(dataset: Dataset, images: Optional[List[ImageRequestModel dataset.images = [] images_updated = False for img in images: - img_object = img.to_image_object(dataset) + image_folder = project_context.dataset_images_path / dataset.initial_identifier + try: + img_object = img.to_image_object(owner_id=dataset.id) + except errors.ImageError as e: + raise errors.DatasetImageError(e) from e + + path = img_object.content_url + + if not img_object.is_remote: + # NOTE: only copy dataset image if it's not in .renku/datasets//images/ already + if not path.startswith(str(image_folder)): + image_type = imghdr.what(path) + if image_type: + ext = f".{image_type}" + else: + _, ext = os.path.splitext(path) + target_image_path: Union[Path, str] = image_folder / f"{img_object.position}{ext}" - if not img_object: - continue + image_folder.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(path, target_image_path) + else: + target_image_path = path + + img_object.content_url = get_relative_path(target_image_path, base=project_context.path) # type: ignore if any(i.position == img_object.position for i in dataset.images): raise errors.DatasetImageError(f"Duplicate dataset image specified for position {img_object.position}") diff --git a/renku/core/dataset/request_model.py b/renku/core/dataset/request_model.py deleted file mode 100644 index 1b27c8d72b..0000000000 --- a/renku/core/dataset/request_model.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright Swiss Data Science Center (SDSC). A partnership between -# École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Renku management dataset request models.""" - -import imghdr -import os -import shutil -import urllib -from pathlib import Path -from typing import List, Optional, Union, cast -from urllib.request import urlretrieve - -from renku.core import errors -from renku.domain_model.dataset import Dataset, ImageObject -from renku.domain_model.project_context import project_context - - -class ImageRequestModel: - """Model for passing image information to dataset use-cases.""" - - def __init__( - self, - content_url: str, - position: int, - mirror_locally: bool = False, - safe_image_paths: Optional[List[str]] = None, - ) -> None: - self.content_url = content_url - self.position = position - self.mirror_locally = mirror_locally - self.safe_image_paths: List[Union[str, Path]] = cast(List[Union[str, Path]], safe_image_paths) or [] - - def to_image_object(self, dataset: Dataset) -> ImageObject: - """Convert request model to ``ImageObject``.""" - image_type = None - self.safe_image_paths.append(project_context.path) - - image_folder = project_context.dataset_images_path / dataset.initial_identifier - image_folder.mkdir(exist_ok=True, parents=True) - - if urllib.parse.urlparse(self.content_url).netloc: - # NOTE: absolute url - if not self.mirror_locally: - return ImageObject( - content_url=self.content_url, - position=self.position, - id=ImageObject.generate_id(dataset_id=dataset.id, position=self.position), - ) - - # NOTE: mirror the image locally - try: - path, _ = urlretrieve(self.content_url) - except urllib.error.URLError as e: - raise errors.DatasetImageError(f"Dataset image with url {self.content_url} couldn't be mirrored") from e - - image_type = imghdr.what(path) - if image_type: - image_type = f".{image_type}" - - self.content_url = path - self.safe_image_paths.append(Path(path).parent) - - path = self.content_url - if not os.path.isabs(path): - path = os.path.normpath(os.path.join(project_context.path, path)) - - if not os.path.exists(path) or not any( - os.path.commonprefix([path, p]) == str(p) for p in self.safe_image_paths - ): - # NOTE: make sure files exists and prevent path traversal - raise errors.DatasetImageError(f"Dataset image with relative path {self.content_url} not found") - - if not path.startswith(str(image_folder)): - # NOTE: only copy dataset image if it's not in .renku/datasets//images/ already - if image_type: - ext = image_type - else: - _, ext = os.path.splitext(self.content_url) - - img_path = image_folder / f"{self.position}{ext}" - shutil.copy(path, img_path) - else: - img_path = Path(path) - - return ImageObject( - content_url=str(img_path.relative_to(project_context.path)), - position=self.position, - id=ImageObject.generate_id(dataset_id=dataset.id, position=self.position), - ) diff --git a/renku/core/errors.py b/renku/core/errors.py index 0da28254c1..dae89f752f 100644 --- a/renku/core/errors.py +++ b/renku/core/errors.py @@ -512,7 +512,11 @@ class RenkuSaveError(RenkuException): """Raised when renku save doesn't work.""" -class DatasetImageError(DatasetException): +class ImageError(RenkuException): + """Raised when an image for a project/dataset is not accessible.""" + + +class DatasetImageError(DatasetException, ImageError): """Raised when a local dataset image is not accessible.""" diff --git a/renku/core/image.py b/renku/core/image.py new file mode 100644 index 0000000000..6e875912f1 --- /dev/null +++ b/renku/core/image.py @@ -0,0 +1,82 @@ +# Copyright Swiss Data Science Center (SDSC). A partnership between +# École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Renku project/dataset image management.""" + +import imghdr +import os +import urllib +from pathlib import Path +from typing import List, Optional, Union, cast +from urllib.request import urlretrieve + +from renku.core import errors +from renku.core.constant import FILESYSTEM_ROOT +from renku.core.util.os import is_subpath +from renku.core.util.urls import is_remote +from renku.domain_model.image import ImageObject +from renku.domain_model.project_context import project_context + + +class ImageObjectRequest: + """Model for passing image information.""" + + def __init__( + self, + content_url: str, + position: int = 0, + mirror_locally: bool = True, + safe_image_paths: Optional[List[str]] = None, + ) -> None: + self.content_url = content_url + self.position = position + self.mirror_locally = mirror_locally + self.safe_image_paths: List[Union[str, Path]] = cast(List[Union[str, Path]], safe_image_paths) or [] + + def to_image_object(self, owner_id: str) -> ImageObject: + """Convert request model to ``ImageObject`` and download the image if requested and return its path.""" + self.safe_image_paths.append(project_context.path) + + if is_remote(self.content_url): + if not self.mirror_locally: + return ImageObject( + content_url=self.content_url, + position=self.position, + id=ImageObject.generate_id(owner_id=owner_id, position=self.position), + ) + + # NOTE: Download the image + try: + tmp_path, _ = urlretrieve(self.content_url) + except urllib.error.URLError as e: + raise errors.ImageError(f"Cannot download image with url {self.content_url}: {e}") from e + + path = Path(tmp_path) + else: + path = Path(self.content_url).resolve() + + if not os.path.exists(path): + raise errors.ImageError(f"Image with local path '{self.content_url}' not found") + # NOTE: Prevent path traversal or usage of non-image files + elif (FILESYSTEM_ROOT in self.safe_image_paths and imghdr.what(path) is None) or not any( + is_subpath(path, base=p) for p in self.safe_image_paths + ): + raise errors.ImageError(f"'{self.content_url}' isn't a valid image file") + + return ImageObject( + content_url=path.as_posix(), + position=self.position, + id=ImageObject.generate_id(owner_id=owner_id, position=self.position), + ) diff --git a/renku/core/init.py b/renku/core/init.py index 18f12ef560..f62cd4cc27 100644 --- a/renku/core/init.py +++ b/renku/core/init.py @@ -30,8 +30,10 @@ from renku.core.constant import DATA_DIR_CONFIG_KEY, RENKU_HOME from renku.core.git import with_worktree from renku.core.githooks import install_githooks +from renku.core.image import ImageObjectRequest from renku.core.interface.database_gateway import IDatabaseGateway from renku.core.migration.utils import OLD_METADATA_PATH +from renku.core.project import set_project_image from renku.core.storage import init_external_storage, storage_installed from renku.core.template.template import ( FileAction, @@ -101,6 +103,7 @@ def init_project( name: Optional[str], description: Optional[str], keywords: Optional[List[str]], + image_request: Optional[ImageObjectRequest], template_id: Optional[str], template_source: Optional[str], template_ref: Optional[str], @@ -114,11 +117,12 @@ def init_project( """Initialize a renku project. Args: - external_storage_requested: Whether or not external storage should be used. + external_storage_requested: Whether external storage should be used. path: Path to initialize repository at. name: Name of the project. description: Description of the project. keywords: keywords for the project. + image_request(Optional[ImageObjectRequest]): Project's image. template_id: id of the template to use. template_source: Source to get the template from. template_ref: Reference to use to get the template. @@ -211,6 +215,7 @@ def init_project( description=description, keywords=keywords, install_mergetool=install_mergetool, + image_request=image_request, ) except FileExistsError as e: raise errors.InvalidFileOperation(e) @@ -264,6 +269,7 @@ def create_from_template( commit_message: Optional[str] = None, description: Optional[str] = None, keywords: Optional[List[str]] = None, + image_request: Optional[ImageObjectRequest] = None, install_mergetool: bool = False, ): """Initialize a new project from a template. @@ -278,7 +284,8 @@ def create_from_template( commit_message(Optional[str]): Message for initial commit (Default value = None). description(Optional[str]): Description of the project (Default value = None). keywords(Optional[List[str]]): Keywords for project (Default value = None). - install_mergetool(bool): Whether to setup renku metadata mergetool (Default value = False). + image_request(Optional[ImageObjectRequest]): Project's image (Default value = None). + install_mergetool(bool): Whether to set up renku metadata mergetool (Default value = False). """ commit_only = [f"{RENKU_HOME}/", str(project_context.template_checksums_path)] + list(rendered_template.get_files()) @@ -312,6 +319,9 @@ def create_from_template( ) as project: copy_template_to_project(rendered_template=rendered_template, project=project, actions=actions) + # NOTE: Copy image to project + set_project_image(image_request=image_request) + if install_mergetool: setup_mergetool() @@ -337,6 +347,7 @@ def create_from_template_local( keywords: Optional[List[str]] = None, data_dir: Optional[str] = None, ssh_supported: bool = False, + image_request: Optional[ImageObjectRequest] = None, ): """Initialize a new project from a template. @@ -356,6 +367,7 @@ def create_from_template_local( description(Optional[str]): Project description (Default value = None). keywords(Optional[List[str]]): Project keywords (Default value = None). data_dir(Optional[str]): Project base data directory (Default value = None). + image_request(Optional[ImageObjectRequest]): Project's image (Default value = None). """ metadata = metadata or {} default_metadata = default_metadata or {} @@ -410,4 +422,5 @@ def create_from_template_local( description=description, keywords=keywords, data_dir=data_dir, + image_request=image_request, ) diff --git a/renku/core/migration/m_0010__metadata_fixes.py b/renku/core/migration/m_0010__metadata_fixes.py index 11f7d49035..3cbc04a9bb 100644 --- a/renku/core/migration/m_0010__metadata_fixes.py +++ b/renku/core/migration/m_0010__metadata_fixes.py @@ -238,7 +238,7 @@ def migrate_project_template_data(project_gateway: IProjectGateway): @inject.autoparams("plan_gateway") def fix_plan_times(plan_gateway: IPlanGateway): - """Add timezone to plan invalidations.""" + """Rename plan's date-related attributes and add timezone to invalidation time.""" plans: List[AbstractPlan] = plan_gateway.get_all_plans() for plan in plans: @@ -248,6 +248,8 @@ def fix_plan_times(plan_gateway: IPlanGateway): del plan.invalidated_at elif not hasattr(plan, "date_removed"): plan.date_removed = None + if not hasattr(plan, "date_created"): + plan.date_created = getattr(plan, "date_modified", None) or plan.date_removed or local_now() if plan.date_removed is not None: if plan.date_removed.tzinfo is None: diff --git a/renku/core/migration/utils/conversion.py b/renku/core/migration/utils/conversion.py index 7e86e261aa..13c5f0490d 100644 --- a/renku/core/migration/utils/conversion.py +++ b/renku/core/migration/utils/conversion.py @@ -28,12 +28,12 @@ Dataset, DatasetFile, DatasetTag, - ImageObject, Language, RemoteEntity, Url, is_dataset_name_valid, ) +from renku.domain_model.image import ImageObject from renku.domain_model.project_context import project_context from renku.domain_model.provenance import agent as new_agents @@ -65,7 +65,7 @@ def _convert_image_object(image_object: Optional[old_datasets.ImageObject], data """Create from old ImageObject instance.""" if not image_object: return - id = ImageObject.generate_id(dataset_id=dataset_id, position=image_object.position) + id = ImageObject.generate_id(owner_id=dataset_id, position=image_object.position) return ImageObject(content_url=image_object.content_url, position=image_object.position, id=id) diff --git a/renku/core/project.py b/renku/core/project.py index c852a31ecf..8c50a5ef66 100644 --- a/renku/core/project.py +++ b/renku/core/project.py @@ -15,15 +15,21 @@ # limitations under the License. """Project business logic.""" +import os +import shutil from typing import Dict, List, Optional, Union, cast from pydantic import validate_arguments from renku.command.command_builder import inject from renku.command.view_model.project import ProjectViewModel +from renku.core import errors +from renku.core.image import ImageObjectRequest from renku.core.interface.project_gateway import IProjectGateway from renku.core.util.metadata import construct_creator +from renku.core.util.os import get_relative_path from renku.domain_model.constant import NO_VALUE, NoValueType +from renku.domain_model.dataset import ImageObjectRequestJson from renku.domain_model.project_context import project_context from renku.domain_model.provenance.agent import Person @@ -36,6 +42,7 @@ def edit_project( keywords: Optional[Union[List[str], NoValueType]], custom_metadata: Optional[Union[Dict, List[Dict], NoValueType]], custom_metadata_source: Optional[Union[str, NoValueType]], + image_request: Optional[Union[ImageObjectRequest, NoValueType]], project_gateway: IProjectGateway, ): """Edit dataset metadata. @@ -47,6 +54,7 @@ def edit_project( custom_metadata(Union[Optional[Dict, List[Dict]]): Custom JSON-LD metadata. custom_metadata_source(Optional[str]): Custom metadata source. project_gateway(IProjectGateway): Injected project gateway. + image_request(Optional[ImageObjectRequest]): Project's image. Returns: Tuple of fields that were updated and dictionary of warnings. @@ -56,6 +64,11 @@ def edit_project( "description": description, "keywords": keywords, "custom_metadata": custom_metadata, + "image": NO_VALUE + if image_request is NO_VALUE + else None + if image_request is None + else ImageObjectRequestJson().dump(image_request), } no_email_warnings: Optional[Union[Dict, str]] = None @@ -64,10 +77,16 @@ def edit_project( if creator is not NO_VALUE: parsed_creator, no_email_warnings = construct_creator(cast(Union[Dict, str], creator), ignore_email=True) + if image_request is None: + delete_project_image() + elif image_request is not NO_VALUE: + set_project_image(image_request=image_request) # type: ignore + updated = {k: v for k, v in possible_updates.items() if v is not NO_VALUE} if updated: project = project_gateway.get_project() + # NOTE: No need to pass ``image`` here since we already copied/deleted the file and updated the project project.update_metadata( creator=parsed_creator, description=description, @@ -87,3 +106,47 @@ def show_project() -> ProjectViewModel: Project view model. """ return ProjectViewModel.from_project(project_context.project) + + +def set_project_image(image_request: Optional[ImageObjectRequest]) -> None: + """Download and set a project's images. + + Args: + image_request(Optional[ImageObjectRequest]): The image to set. + """ + if image_request is None: + return + + # NOTE: Projects can have maximum one image + image_request.position = 0 + + image_object = image_request.to_image_object(owner_id=project_context.project.id) + + project_image = project_context.project_image_pathname + + # NOTE: Do nothing if the new path is the same as the old one + if project_image.resolve() != image_object.content_url: + # NOTE: Always delete the old image + delete_project_image() + + if not image_object.is_remote: + project_image.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(image_object.content_url, project_context.project_image_pathname) + + image_object.content_url = get_relative_path(project_image, base=project_context.path) # type: ignore + + image_object.position = 0 + + project_context.project.image = image_object + + +def delete_project_image() -> None: + """Delete project image in a project.""" + try: + os.remove(project_context.project_image_pathname) + except FileNotFoundError: + pass + except OSError as e: + raise errors.ImageError(f"Cannot delete project image '{project_context.project_image_pathname}': {e}") from e + else: + project_context.project.image = None diff --git a/renku/core/util/contexts.py b/renku/core/util/contexts.py index d9cdf8a7c3..9f3599c4cf 100644 --- a/renku/core/util/contexts.py +++ b/renku/core/util/contexts.py @@ -162,10 +162,14 @@ def with_project_metadata( custom_metadata=custom_metadata, ) - yield project - - if not read_only: + if read_only: + yield project + else: + # NOTE: Set project so that ``project_context`` can be used inside the code project_gateway.update_project(project) + + yield project + database_gateway.commit() diff --git a/renku/core/util/urls.py b/renku/core/util/urls.py index 9a491d58af..a5bacdb404 100644 --- a/renku/core/util/urls.py +++ b/renku/core/util/urls.py @@ -167,3 +167,9 @@ def check_url(url: str) -> Tuple[bool, bool]: is_git = is_remote and (u.path.lower().endswith(".git") or scheme in ("git+https", "git+ssh") or starts_with_git) return is_remote, is_git + + +def is_remote(uri: str) -> bool: + """Returns True if a given URI is remote.""" + is_remote, _ = check_url(uri) + return is_remote diff --git a/renku/data/shacl_shape.json b/renku/data/shacl_shape.json index f52cc3f4fb..fb2f78c9d5 100644 --- a/renku/data/shacl_shape.json +++ b/renku/data/shacl_shape.json @@ -159,6 +159,14 @@ } ] }, + { + "path": "schema:image", + "sh:class": { + "@id": "schema:ImageObject" + }, + "sh:pattern": "http(s)?://[^/]+/projects/.+/images/0", + "maxCount": 1 + }, { "nodeKind": "sh:Literal", "path": "renku:templateSource", diff --git a/renku/domain_model/dataset.py b/renku/domain_model/dataset.py index 8d234241ff..8b68d056ff 100644 --- a/renku/domain_model/dataset.py +++ b/renku/domain_model/dataset.py @@ -32,8 +32,9 @@ from renku.core.util.git import get_entity_from_revision from renku.core.util.metadata import is_linked_file from renku.core.util.os import get_absolute_path -from renku.core.util.urls import get_path, get_slug +from renku.core.util.urls import get_slug from renku.domain_model.constant import NO_VALUE, NON_EXISTING_ENTITY_CHECKSUM +from renku.domain_model.image import ImageObject from renku.domain_model.project_context import project_context from renku.infrastructure.immutable import Immutable, Slots from renku.infrastructure.persistent import Persistent @@ -173,30 +174,6 @@ def generate_id(name: str) -> str: return f"/languages/{name}" -class ImageObject(Slots): - """Represents a schema.org `ImageObject`.""" - - __slots__ = ("content_url", "id", "position") - - id: str - content_url: str - position: int - - def __init__(self, *, content_url: str, id: str, position: int): - id = get_path(id) - super().__init__(content_url=content_url, position=position, id=id) - - @staticmethod - def generate_id(dataset_id: str, position: int) -> str: - """Generate @id field.""" - return f"{dataset_id}/images/{position}" - - @property - def is_absolute(self): - """Whether content_url is an absolute or relative url.""" - return bool(urlparse(self.content_url).netloc) - - class RemoteEntity(Slots): """Reference to an Entity in a remote repository.""" @@ -790,8 +767,8 @@ class ImageObjectRequestJson(marshmallow.Schema): file_id = marshmallow.fields.String() content_url = marshmallow.fields.String() - position = marshmallow.fields.Integer() - mirror_locally = marshmallow.fields.Bool(dump_default=False) + position = marshmallow.fields.Integer(load_default=0) + mirror_locally = marshmallow.fields.Bool(load_default=False) def get_file_path_in_dataset(dataset: Dataset, dataset_file: DatasetFile) -> Path: diff --git a/renku/domain_model/image.py b/renku/domain_model/image.py new file mode 100644 index 0000000000..87b0483113 --- /dev/null +++ b/renku/domain_model/image.py @@ -0,0 +1,52 @@ +# Copyright Swiss Data Science Center (SDSC). A partnership between +# École Polytechnique Fédérale de Lausanne (EPFL) and +# Eidgenössische Technische Hochschule Zürich (ETHZ). +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Image model.""" + +from pathlib import Path +from typing import Union +from urllib.parse import urlparse + +from renku.core.util.urls import get_path, is_remote +from renku.infrastructure.immutable import Slots + + +class ImageObject(Slots): + """Represents a schema.org ``ImageObject``.""" + + __slots__ = ("content_url", "id", "position") + + id: str + content_url: str + position: int + + def __init__(self, *, content_url: Union[str, Path], id: str, position: int): + id = get_path(id) + super().__init__(content_url=str(content_url), position=position, id=id) + + @staticmethod + def generate_id(owner_id: str, position: int) -> str: + """Generate @id field.""" + return f"{owner_id}/images/{position}" + + @property + def is_absolute(self): + """Whether content_url is an absolute or relative url.""" + return bool(urlparse(self.content_url).netloc) + + @property + def is_remote(self) -> bool: + """Return True if the URI isn't on the local filesystem.""" + return is_remote(self.content_url) diff --git a/renku/domain_model/project.py b/renku/domain_model/project.py index 397fadda22..52b3ddd697 100644 --- a/renku/domain_model/project.py +++ b/renku/domain_model/project.py @@ -27,6 +27,7 @@ from renku.core.util.git import get_git_user from renku.core.util.os import normalize_to_ascii from renku.domain_model.constant import NO_VALUE +from renku.domain_model.image import ImageObject from renku.domain_model.provenance.agent import Person from renku.domain_model.provenance.annotation import Annotation from renku.version import __minimum_project_version__ @@ -53,6 +54,7 @@ class Project(persistent.Persistent): """Represent a project.""" keywords: List[str] = list() + image: Optional[ImageObject] = None # NOTE: the minimum version of renku to needed to work with a project # This should be bumped on metadata version changes and when we do not forward-compatible on-the-fly migrations @@ -71,6 +73,7 @@ def __init__( template_metadata: Optional[ProjectTemplateMetadata] = None, version: Optional[str] = None, keywords: Optional[List[str]] = None, + image: Optional[ImageObject] = None, ): from renku.core.migration.migrate import SUPPORTED_PROJECT_VERSION @@ -91,6 +94,7 @@ def __init__( self.id: str = id self.version: str = version self.keywords = keywords or [] + self.image = image self.template_metadata: ProjectTemplateMetadata = template_metadata or ProjectTemplateMetadata() @@ -107,6 +111,7 @@ def from_project_context( keywords: Optional[List[str]] = None, custom_metadata: Optional[Dict] = None, creator: Optional[Person] = None, + image: Optional[ImageObject] = None, ) -> "Project": """Create an instance from a path. @@ -119,6 +124,7 @@ def from_project_context( custom_metadata(Optional[Dict]): Custom JSON-LD metadata (when creating a new project) (Default value = None). creator(Optional[Person]): The project creator. + image(Optional[ImageObject]): Project's image/avatar. """ creator = creator or get_git_user(repository=project_context.repository) @@ -141,7 +147,13 @@ def from_project_context( id = cls.generate_id(namespace=namespace, name=name) return cls( - creator=creator, id=id, name=name, description=description, keywords=keywords, annotations=annotations + annotations=annotations, + creator=creator, + description=description, + id=id, + image=image, + keywords=keywords, + name=name, ) @staticmethod diff --git a/renku/domain_model/project_context.py b/renku/domain_model/project_context.py index c545b40752..bc862b7409 100644 --- a/renku/domain_model/project_context.py +++ b/renku/domain_model/project_context.py @@ -31,6 +31,7 @@ DATASET_IMAGES, DEFAULT_DATA_DIR, DOCKERFILE, + IMAGES, LOCK_SUFFIX, POINTERS, RENKU_HOME, @@ -92,6 +93,11 @@ def dataset_images_path(self) -> Path: """Return a ``Path`` instance of Renku dataset metadata folder.""" return self.path / RENKU_HOME / DATASET_IMAGES + @property + def project_image_pathname(self) -> Path: + """Return the path to the project's image file.""" + return self.path / RENKU_HOME / IMAGES / "project" / "0.png" + @property def dockerfile_path(self) -> Path: """Path to the Dockerfile.""" diff --git a/renku/domain_model/sort.py b/renku/domain_model/sort.py deleted file mode 100644 index 85203083ac..0000000000 --- a/renku/domain_model/sort.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright Swiss Data Science Center (SDSC). A partnership between -# École Polytechnique Fédérale de Lausanne (EPFL) and -# Eidgenössische Technische Hochschule Zürich (ETHZ). -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Process Git repository.""" - -from collections import deque -from typing import Any, Deque - -GRAY, BLACK = 0, 1 - - -def topological(nodes): - """Return nodes in a topological order.""" - order: Deque[Any] - order, enter, state = deque(), set(nodes), {} - - def dfs(node): - """Visit nodes in depth-first order.""" - state[node] = GRAY - for parent in nodes.get(node, ()): - color = state.get(parent, None) - if color == GRAY: - raise ValueError("cycle") - if color == BLACK: - continue - enter.discard(parent) - dfs(parent) - order.appendleft(node) - state[node] = BLACK - - while enter: - dfs(enter.pop()) - - return order diff --git a/renku/domain_model/workflow/plan.py b/renku/domain_model/workflow/plan.py index a115b5be5f..95b179f200 100644 --- a/renku/domain_model/workflow/plan.py +++ b/renku/domain_model/workflow/plan.py @@ -374,7 +374,7 @@ def set_parameters_from_strings(self, params_strings: List[str]) -> None: def copy(self): """Create a copy of this plan. - Required where a plan is used several times in a workflow but we need to set different values on them. + Required where a plan is used several times in a workflow, but we need to set different values on them. """ return copy.deepcopy(self) diff --git a/renku/ui/cli/init.py b/renku/ui/cli/init.py index 01d8ffcf6c..3dffea2d52 100644 --- a/renku/ui/cli/init.py +++ b/renku/ui/cli/init.py @@ -231,6 +231,7 @@ def resolve_data_directory(data_dir, path): type=click.Path(writable=True, file_okay=False), help="Data directory within the project", ) +@click.option("-i", "--image", default=None, type=str, help="Path or URL to project's avatar/image.") @click.option("-t", "--template-id", help="Provide the id of the template to use.") @click.option("-s", "--template-source", help="Provide the templates repository url or path.") @click.option( @@ -268,6 +269,7 @@ def init( path, name, description, + image, keyword, template_id, template_source, @@ -282,6 +284,8 @@ def init( ): """Initialize a project in PATH. Default is the current path.""" from renku.command.init import init_project_command + from renku.core.constant import FILESYSTEM_ROOT + from renku.core.image import ImageObjectRequest from renku.core.util.git import check_global_git_user_is_configured from renku.ui.cli.utils.callback import ClickCallback @@ -300,6 +304,8 @@ def init( if metadata: custom_metadata = json.loads(Path(metadata).read_text()) + image_request = ImageObjectRequest(content_url=image, safe_image_paths=[FILESYSTEM_ROOT]) if image else None + communicator = ClickCallback() init_project_command().with_communicator(communicator).build().execute( external_storage_requested=external_storage_requested, @@ -307,6 +313,7 @@ def init( name=name, description=description, keywords=keyword, + image_request=image_request, template_id=template_id, template_source=template_source, template_ref=template_ref, diff --git a/renku/ui/cli/project.py b/renku/ui/cli/project.py index 695f9e6447..7e4834bea8 100644 --- a/renku/ui/cli/project.py +++ b/renku/ui/cli/project.py @@ -33,11 +33,12 @@ import json from pathlib import Path +from typing import Optional, Union import click import renku.ui.cli.utils.color as color -from renku.domain_model.constant import NO_VALUE +from renku.domain_model.constant import NO_VALUE, NoValueType from renku.ui.cli.utils.callback import ClickCallback @@ -59,6 +60,7 @@ def project(): type=click.UNPROCESSED, help="Creator's name, email, and affiliation. Accepted format is 'Forename Surname [affiliation]'.", ) +@click.option("-i", "--image", default=NO_VALUE, type=click.UNPROCESSED, help="Path or URL to project's avatar/image.") @click.option( "-m", "--metadata", @@ -71,8 +73,8 @@ def project(): "--unset", default=[], multiple=True, - type=click.Choice(["keywords", "k", "metadata", "m"]), - help="Remove keywords from dataset.", + type=click.Choice(["keywords", "k", "metadata", "m", "description", "d", "image", "i"]), + help="Remove keywords, metadata, description, or image from project.", ) @click.option( "--metadata-source", @@ -80,9 +82,11 @@ def project(): default=NO_VALUE, help="Set the source field in the metadata when editing it if not provided, then the default is 'renku'.", ) -def edit(description, keywords, creators, metadata, unset, metadata_source): +def edit(description, keywords, creators, image, metadata, unset, metadata_source): """Edit project metadata.""" from renku.command.project import edit_project_command + from renku.core.constant import FILESYSTEM_ROOT + from renku.core.image import ImageObjectRequest if list(creators) == [NO_VALUE]: creators = NO_VALUE @@ -100,6 +104,20 @@ def edit(description, keywords, creators, metadata, unset, metadata_source): raise click.UsageError("Cant use '--metadata' together with unsetting metadata") metadata = None + if "d" in unset or "description" in unset: + if description is not NO_VALUE: + raise click.UsageError("Cant use '--description' together with unsetting description") + description = None + + if "i" in unset or "image" in unset: + if image is not NO_VALUE: + raise click.UsageError("Cant use '--image' together with unsetting image") + image_request: Optional[Union[ImageObjectRequest, NoValueType]] = None + elif image is not NO_VALUE: + image_request = ImageObjectRequest(content_url=image, safe_image_paths=[FILESYSTEM_ROOT]) + else: + image_request = NO_VALUE + if metadata_source is not NO_VALUE and metadata is NO_VALUE: raise click.UsageError("The '--metadata-source' option can only be used with the '--metadata' flag") @@ -122,6 +140,7 @@ def edit(description, keywords, creators, metadata, unset, metadata_source): description=description, creator=creators, keywords=keywords, + image_request=image_request, custom_metadata=custom_metadata, custom_metadata_source=metadata_source, ) diff --git a/renku/ui/service/controllers/datasets_create.py b/renku/ui/service/controllers/datasets_create.py index aaa47a5cb1..694282a699 100644 --- a/renku/ui/service/controllers/datasets_create.py +++ b/renku/ui/service/controllers/datasets_create.py @@ -16,7 +16,7 @@ # limitations under the License. """Renku service datasets create controller.""" from renku.command.dataset import create_dataset_command -from renku.core.dataset.request_model import ImageRequestModel +from renku.core.image import ImageObjectRequest from renku.core.util.metadata import construct_creators from renku.ui.service.cache.models.job import Job from renku.ui.service.config import CACHE_UPLOADS_PATH, MESSAGE_PREFIX @@ -53,7 +53,7 @@ def renku_op(self): set_url_for_uploaded_images(images=images, cache=self.cache, user=self.user) images = [ - ImageRequestModel( + ImageObjectRequest( content_url=img["content_url"], position=img["position"], mirror_locally=img.get("mirror_locally", False), diff --git a/renku/ui/service/controllers/datasets_edit.py b/renku/ui/service/controllers/datasets_edit.py index ed50999b7b..e751352da4 100644 --- a/renku/ui/service/controllers/datasets_edit.py +++ b/renku/ui/service/controllers/datasets_edit.py @@ -18,7 +18,7 @@ from typing import Dict, List, Union, cast from renku.command.dataset import edit_dataset_command -from renku.core.dataset.request_model import ImageRequestModel +from renku.core.image import ImageObjectRequest from renku.core.util.metadata import construct_creators from renku.domain_model.constant import NO_VALUE, NoValueType from renku.domain_model.provenance.agent import Person @@ -61,7 +61,7 @@ def renku_op(self): set_url_for_uploaded_images(images=images, cache=self.cache, user=self.user) images = [ - ImageRequestModel( + ImageObjectRequest( content_url=img["content_url"], position=img["position"], mirror_locally=img.get("mirror_locally", False), diff --git a/renku/ui/service/controllers/project_edit.py b/renku/ui/service/controllers/project_edit.py index a07a14e9b7..897a353465 100644 --- a/renku/ui/service/controllers/project_edit.py +++ b/renku/ui/service/controllers/project_edit.py @@ -15,13 +15,17 @@ # See the License for the specific language governing permissions and # limitations under the License. """Renku service project edit controller.""" -from typing import Dict, cast + +from typing import Dict, Optional, Union, cast from renku.command.project import edit_project_command -from renku.domain_model.constant import NO_VALUE +from renku.core.image import ImageObjectRequest +from renku.domain_model.constant import NO_VALUE, NoValueType from renku.ui.service.cache.models.job import Job +from renku.ui.service.config import CACHE_UPLOADS_PATH from renku.ui.service.controllers.api.abstract import ServiceCtrl from renku.ui.service.controllers.api.mixins import RenkuOpSyncMixin +from renku.ui.service.controllers.utils.datasets import set_url_for_uploaded_images from renku.ui.service.serializers.project import ProjectEditRequest, ProjectEditResponseRPC from renku.ui.service.views import result_response @@ -76,6 +80,24 @@ def renku_op(self): else: keywords = NO_VALUE + if "image" not in self.ctx: + image_request: Optional[Union[ImageObjectRequest, NoValueType]] = NO_VALUE + elif self.ctx["image"] is None: + image_request = None + else: + image: Dict = self.ctx.get("image") # type: ignore + + user_cache_dir = CACHE_UPLOADS_PATH / self.user.user_id + + set_url_for_uploaded_images(images=[image], cache=self.cache, user=self.user) + + image_request = ImageObjectRequest( + content_url=image["content_url"], + position=0, + mirror_locally=image.get("mirror_locally", False), + safe_image_paths=[user_cache_dir], + ) + result = ( edit_project_command() .with_commit_message(self.ctx["commit_message"]) @@ -86,6 +108,7 @@ def renku_op(self): custom_metadata=custom_metadata, custom_metadata_source=custom_metadata_source, keywords=keywords, + image_request=image_request, ) ) diff --git a/renku/ui/service/controllers/templates_create_project.py b/renku/ui/service/controllers/templates_create_project.py index b3dc30e753..23712fecff 100644 --- a/renku/ui/service/controllers/templates_create_project.py +++ b/renku/ui/service/controllers/templates_create_project.py @@ -23,13 +23,15 @@ from renku.command.init import create_from_template_local_command from renku.core import errors +from renku.core.image import ImageObjectRequest from renku.core.template.template import fetch_templates_source from renku.core.util.contexts import renku_project_context from renku.domain_model.template import Template from renku.infrastructure.repository import Repository -from renku.ui.service.config import MESSAGE_PREFIX +from renku.ui.service.config import CACHE_UPLOADS_PATH, MESSAGE_PREFIX from renku.ui.service.controllers.api.abstract import ServiceCtrl from renku.ui.service.controllers.api.mixins import RenkuOperationMixin +from renku.ui.service.controllers.utils.datasets import set_url_for_uploaded_images from renku.ui.service.errors import UserProjectCreationError, UserTemplateInvalidError from renku.ui.service.serializers.templates import ProjectTemplateRequest, ProjectTemplateResponseRPC from renku.ui.service.utils import new_repo_push @@ -102,6 +104,7 @@ def setup_new_project(self): "owner": self.ctx["project_namespace"], "token": self.ctx["token"], "initialized": True, + "image": self.ctx["image"], } project = self.cache.make_project(self.user, new_project_data) @@ -150,6 +153,18 @@ def new_project(self): new_project = self.setup_new_project() new_project_path = new_project.abs_path + image = self.ctx.get("image") + if image: + user_cache_dir = CACHE_UPLOADS_PATH / self.user.user_id + set_url_for_uploaded_images(images=[image], cache=self.cache, user=self.user) + + image = ImageObjectRequest( + content_url=image["content_url"], + position=0, + mirror_locally=image.get("mirror_locally", False), + safe_image_paths=[user_cache_dir], + ) + with renku_project_context(new_project_path): create_from_template_local_command().build().execute( self.template.path, @@ -167,6 +182,7 @@ def new_project(self): description=self.ctx["project_description"], data_dir=self.ctx.get("data_directory"), ssh_supported=self.template.ssh_supported, + image_request=image, ) self.new_project_push(new_project_path) diff --git a/renku/ui/service/serializers/datasets.py b/renku/ui/service/serializers/datasets.py index a75569ae6a..c25537ec6f 100644 --- a/renku/ui/service/serializers/datasets.py +++ b/renku/ui/service/serializers/datasets.py @@ -20,7 +20,7 @@ from renku.domain_model.dataset import DatasetCreatorsJson as DatasetCreators from renku.domain_model.dataset import DatasetDetailsJson as DatasetDetails from renku.domain_model.dataset import ImageObjectJson as ImageObject -from renku.domain_model.dataset import ImageObjectRequestJson as ImageObjectRequest +from renku.domain_model.dataset import ImageObjectRequestJson from renku.ui.service.serializers.common import ( AsyncSchema, JobDetailsResponse, @@ -40,7 +40,7 @@ class DatasetNameSchema(Schema): class DatasetDetailsRequest(DatasetDetails): """Request schema with dataset image information.""" - images = fields.List(fields.Nested(ImageObjectRequest)) + images = fields.List(fields.Nested(ImageObjectRequestJson)) custom_metadata: fields.Field = fields.Dict() @@ -200,7 +200,7 @@ class DatasetEditRequest( creators = fields.List(fields.Nested(DatasetCreators), metadata={"description": "New creators of the dataset"}) keywords = fields.List(fields.String(), allow_none=True, metadata={"description": "New keywords for the dataset"}) images = fields.List( - fields.Nested(ImageObjectRequest), allow_none=True, metadata={"description": "New dataset images"} + fields.Nested(ImageObjectRequestJson), allow_none=True, metadata={"description": "New dataset images"} ) custom_metadata = fields.List( fields.Dict(), metadata={"description": "New custom metadata for the dataset"}, allow_none=True diff --git a/renku/ui/service/serializers/project.py b/renku/ui/service/serializers/project.py index e93fa1a41d..62e167ebcf 100644 --- a/renku/ui/service/serializers/project.py +++ b/renku/ui/service/serializers/project.py @@ -19,6 +19,7 @@ from marshmallow.schema import Schema from renku.domain_model.dataset import DatasetCreatorsJson as DatasetCreators +from renku.domain_model.dataset import ImageObjectRequestJson from renku.ui.service.serializers.common import ( AsyncSchema, MigrateSchema, @@ -78,6 +79,7 @@ class ProjectEditRequest(AsyncSchema, RemoteRepositorySchema, MigrateSchema): metadata={"description": "The source for the JSON-LD metadata"}, ) keywords = fields.List(fields.String(), allow_none=True, metadata={"description": "New keyword(s) for the project"}) + image = fields.Nested(ImageObjectRequestJson, allow_none=True, metadata={"description": "Image for the project"}) class ProjectEditResponse(RenkuSyncSchema): diff --git a/renku/ui/service/serializers/templates.py b/renku/ui/service/serializers/templates.py index 783f5ca519..3dd7096ac3 100644 --- a/renku/ui/service/serializers/templates.py +++ b/renku/ui/service/serializers/templates.py @@ -23,6 +23,7 @@ from yagup.exceptions import InvalidURL from renku.core.util.os import normalize_to_ascii +from renku.domain_model.dataset import ImageObjectRequestJson from renku.ui.service.config import TEMPLATE_CLONE_DEPTH_DEFAULT from renku.ui.service.serializers.cache import ProjectCloneContext, RepositoryCloneRequest from renku.ui.service.serializers.rpc import JsonRPCResponse @@ -70,6 +71,7 @@ class ProjectTemplateRequest(ProjectCloneContext, ManifestTemplatesRequest): data_directory = fields.String( load_default=None, metadata={"description": "Base dataset data directory in project. Defaults to 'data/'"} ) + image = fields.Nested(ImageObjectRequestJson, load_default=None) @post_load() def add_required_fields(self, data, **kwargs): diff --git a/renku/ui/service/views/error_handlers.py b/renku/ui/service/views/error_handlers.py index 0e828d6f0c..5babc2d8ad 100644 --- a/renku/ui/service/views/error_handlers.py +++ b/renku/ui/service/views/error_handlers.py @@ -360,7 +360,7 @@ def decorated_function(*args, **kwargs): error_message = str(e) if "Duplicate dataset image" in error_message: raise UserDatasetsMultipleImagesError(e) - elif "couldn't be mirrored" in error_message: + elif "Cannot download image with url" in error_message: raise UserDatasetsUnreachableImageError(e) raise except ValidationError as e: diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 12538aeeee..6ccb7257a9 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -471,3 +471,45 @@ def test_init_with_description(isolated_runner, template): result = isolated_runner.invoke(cli, ["graph", "export", "--strict"]) assert 0 == result.exit_code, format_result_exception(result) assert "my project description" in result.output + + +@pytest.mark.parametrize( + "image", + [Path(__file__).parent / ".." / "data" / "renku.png", "https://en.wikipedia.org/static/images/icons/wikipedia.png"], +) +def test_init_with_image(isolated_runner, template, image): + """Test project initialization with image.""" + result = isolated_runner.invoke(cli, ["init", "--image", image, "new-project", "-t", template["id"]]) + + assert 0 == result.exit_code, format_result_exception(result) + + project = Database.from_path(Path("new-project") / ".renku" / "metadata").get("project") + + assert (Path("new-project") / ".renku" / "images" / "project" / "0.png").exists() + assert ".renku/images/project/0.png" == project.image.content_url + assert 0 == project.image.position + + +@pytest.mark.parametrize( + "image, error", + [ + ("non-existing.png", "Image with local path"), + ("/non-existing", "Image with local path"), + ("https://example.com/non-existing.png", "Cannot download image with url"), + ], +) +def test_init_with_non_existing_image(isolated_runner, template, image, error): + """Test project initialization fails when the image doesn't exist.""" + result = isolated_runner.invoke(cli, ["init", "--image", image, "new-project", "-t", template["id"]]) + + assert 1 == result.exit_code, format_result_exception(result) + assert error in result.output + + +@pytest.mark.parametrize("image", [Path(__file__).parent / ".." / "data" / "workflow-file.yml", "/etc/passwd"]) +def test_init_with_non_image_files(isolated_runner, template, image): + """Test project initialization fails when the file isn't an image format.""" + result = isolated_runner.invoke(cli, ["init", "--image", image, "new-project", "-t", template["id"]]) + + assert 1 == result.exit_code, format_result_exception(result) + assert "isn't a valid image file" in result.output diff --git a/tests/cli/test_integration_datasets.py b/tests/cli/test_integration_datasets.py index c6974d81c9..191eabaee5 100644 --- a/tests/cli/test_integration_datasets.py +++ b/tests/cli/test_integration_datasets.py @@ -208,7 +208,7 @@ def test_dataset_import_real_doi_warnings(runner, project, sleep_after): result = runner.invoke(cli, ["dataset", "ls"]) assert 0 == result.exit_code, format_result_exception(result) + str(result.stderr_bytes) - assert "pyndl_naive_discr_v1.1.1" in result.output + assert "pyndl_naive_discr_v1.1.2" in result.output @pytest.mark.parametrize( diff --git a/tests/cli/test_migrate.py b/tests/cli/test_migrate.py index 3ba1158662..16243fc9fd 100644 --- a/tests/cli/test_migrate.py +++ b/tests/cli/test_migrate.py @@ -371,7 +371,7 @@ def test_commands_fail_on_old_repository(isolated_runner, old_repository_with_su ["doctor"], ["githooks", "install"], ["help"], - ["init", "-i", "1", "--force"], + ["init", "-t", "python-minimal", "--force"], ["storage", "check"], ], ) diff --git a/tests/cli/test_project.py b/tests/cli/test_project.py index 25a46f6b99..9aecff17d9 100644 --- a/tests/cli/test_project.py +++ b/tests/cli/test_project.py @@ -17,6 +17,7 @@ """Test ``project`` command.""" import json +from pathlib import Path import pytest @@ -68,17 +69,21 @@ def test_project_edit(runner, project, subdirectory, with_injection): "keyword1", "-k", "keyword2", + "--image", + Path(__file__).parent / ".." / "data" / "renku.png", ], ) assert 0 == result.exit_code, format_result_exception(result) - assert "Successfully updated: creator, description, keywords, custom_metadata." in result.output + assert "Successfully updated: creator, description, keywords, custom_metadata, image." in result.output assert "Warning: No email or wrong format for: Forename Surname" in result.output assert project.repository.is_dirty() commit_sha_after = project.repository.head.commit.hexsha assert commit_sha_before != commit_sha_after + assert (project.path / ".renku" / "images" / "project" / "0.png").exists() + with with_injection(): project_gateway = ProjectGateway() project = project_gateway.get_project() @@ -99,6 +104,9 @@ def test_project_edit(runner, project, subdirectory, with_injection): assert "Renku Version:" in result.output assert "Keywords:" in result.output + assert ".renku/images/project/0.png" == project.image.content_url + assert 0 == project.image.position + result = runner.invoke(cli, ["graph", "export", "--format", "json-ld", "--strict"]) assert 0 == result.exit_code, format_result_exception(result) @@ -148,27 +156,33 @@ def test_project_edit_unset(runner, project, subdirectory, with_injection): "keyword1", "-k", "keyword2", + "--image", + Path(__file__).parent / ".." / "data" / "renku.png", ], ) assert 0 == result.exit_code, format_result_exception(result) - assert "Successfully updated: creator, description, keywords, custom_metadata." in result.output + assert "Successfully updated: creator, description, keywords, custom_metadata, image." in result.output assert "Warning: No email or wrong format for: Forename Surname" in result.output commit_sha_before = project.repository.head.commit.hexsha + assert (project.path / ".renku" / "images" / "project" / "0.png").exists() + result = runner.invoke( cli, - ["project", "edit", "-u", "keywords", "-u", "metadata"], + ["project", "edit", "-u", "keywords", "-u", "metadata", "-u", "image"], ) assert 0 == result.exit_code, format_result_exception(result) - assert "Successfully updated: keywords, custom_metadata." in result.output + assert "Successfully updated: keywords, custom_metadata, image." in result.output assert project.repository.is_dirty() commit_sha_after = project.repository.head.commit.hexsha assert commit_sha_before != commit_sha_after + assert not (project.path / ".renku" / "images" / "project" / "0.png").exists() + with with_injection(): project_gateway = ProjectGateway() project = project_gateway.get_project() @@ -185,6 +199,8 @@ def test_project_edit_unset(runner, project, subdirectory, with_injection): assert "Renku Version:" in result.output assert "Keywords:" in result.output + assert project.image is None + result = runner.invoke(cli, ["graph", "export", "--format", "json-ld", "--strict"]) assert 0 == result.exit_code, format_result_exception(result) diff --git a/tests/core/commands/test_graph.py b/tests/core/commands/test_graph.py index 65ca13a124..1149d5003d 100644 --- a/tests/core/commands/test_graph.py +++ b/tests/core/commands/test_graph.py @@ -336,7 +336,10 @@ def test_graph_export_full(): project_gateway = MagicMock(spec=IProjectGateway) project_gateway.get_project.return_value = MagicMock( - spec=Project, id="/projects/my-project", date_created=datetime.fromisoformat("2022-07-12T16:29:14+02:00") + spec=Project, + id="/projects/my-project", + date_created=datetime.fromisoformat("2022-07-12T16:29:14+02:00"), + image=None, ) result = get_graph_for_all_objects( diff --git a/tests/core/fixtures/core_database.py b/tests/core/fixtures/core_database.py index 95036003a4..6ad79180a0 100644 --- a/tests/core/fixtures/core_database.py +++ b/tests/core/fixtures/core_database.py @@ -78,7 +78,7 @@ def database() -> Iterator[Tuple["Database", DummyStorage]]: @pytest.fixture def with_injection(): - """Factory fixture for test injections manager.""" + """Factory fixture for test injection manager.""" from renku.command.command_builder.command import inject, remove_injector from renku.domain_model.project_context import project_context diff --git a/tests/data/renku.png b/tests/data/renku.png new file mode 100644 index 0000000000..b3518b1a18 Binary files /dev/null and b/tests/data/renku.png differ diff --git a/tests/service/views/test_exceptions.py b/tests/service/views/test_exceptions.py index e6f96822da..868bd780b0 100644 --- a/tests/service/views/test_exceptions.py +++ b/tests/service/views/test_exceptions.py @@ -184,7 +184,7 @@ def test_project_no_commits(svc_client, it_no_commit_repo_url, identity_headers) @pytest.mark.service @pytest.mark.integration -# @retry_failed +@retry_failed @pytest.mark.parametrize( "git_url", [ diff --git a/tests/service/views/test_project_views.py b/tests/service/views/test_project_views.py index 5c256220ee..8bb0177b13 100644 --- a/tests/service/views/test_project_views.py +++ b/tests/service/views/test_project_views.py @@ -92,6 +92,7 @@ def test_edit_project_view(svc_client_with_repo, custom_metadata, custom_metadat "description": "my new title", "creator": {"name": "name123", "email": "name123@ethz.ch", "affiliation": "ethz"}, "custom_metadata": custom_metadata, + "image": {"content_url": "https://en.wikipedia.org/static/images/icons/wikipedia.png"}, } if custom_metadata_source is not None: edit_payload["custom_metadata_source"] = custom_metadata_source @@ -103,6 +104,11 @@ def test_edit_project_view(svc_client_with_repo, custom_metadata, custom_metadat "description": "my new title", "creator": {"name": "name123", "email": "name123@ethz.ch", "affiliation": "ethz"}, "custom_metadata": custom_metadata, + "image": { + "content_url": "https://en.wikipedia.org/static/images/icons/wikipedia.png", + "mirror_locally": False, + "position": 0, + }, } == response.json["result"]["edited"] edit_payload = { @@ -112,7 +118,7 @@ def test_edit_project_view(svc_client_with_repo, custom_metadata, custom_metadat assert_rpc_response(response) assert {"warning", "edited", "remote_branch"} == set(response.json["result"]) - assert 0 == len(response.json["result"]["edited"]) + assert 0 == len(response.json["result"]["edited"]), response.json["result"]["edited"] @pytest.mark.service @@ -135,10 +141,11 @@ def test_edit_project_view_unset(svc_client_with_repo): "https://schema.org/property2": "test", } ], + "image": {"content_url": "https://en.wikipedia.org/static/images/icons/wikipedia.png"}, } - response = svc_client.post("/project.edit", data=json.dumps(edit_payload), headers=headers) + svc_client.post("/project.edit", data=json.dumps(edit_payload), headers=headers) - edit_payload = {"git_url": url_components.href, "custom_metadata": None, "keywords": None} + edit_payload = {"git_url": url_components.href, "custom_metadata": None, "keywords": None, "image": None} response = svc_client.post("/project.edit", data=json.dumps(edit_payload), headers=headers) assert_rpc_response(response) @@ -146,6 +153,7 @@ def test_edit_project_view_unset(svc_client_with_repo): assert { "keywords": None, "custom_metadata": None, + "image": None, } == response.json[ "result" ]["edited"] diff --git a/tests/service/views/test_templates_views.py b/tests/service/views/test_templates_views.py index 3e26292fc5..e066b8a889 100644 --- a/tests/service/views/test_templates_views.py +++ b/tests/service/views/test_templates_views.py @@ -131,6 +131,10 @@ def test_create_project_from_template(svc_client_templates_creation, with_inject svc_client, headers, payload, rm_remote = svc_client_templates_creation payload["data_directory"] = "my-folder/" + payload["image"] = { + "content_url": "https://en.wikipedia.org/static/images/icons/wikipedia.png", + "mirror_locally": True, + } response = svc_client.post("/templates.create_project", data=json.dumps(payload), headers=headers) @@ -163,6 +167,8 @@ def test_create_project_from_template(svc_client_templates_creation, with_inject assert old_metadata_path.exists() assert "'http://schema.org/schemaVersion': '9'" in old_metadata_path.read_text() + assert (project_path / ".renku" / "images" / "project" / "0.png").exists() + # NOTE: successfully re-use old name after cleanup assert rm_remote() is True sleep(1) # NOTE: sleep to make sure remote isn't locked