diff --git a/.env.sample b/.env.sample index 2ec4a7b8c9a..d83a7bfdbf1 100644 --- a/.env.sample +++ b/.env.sample @@ -168,6 +168,7 @@ SECRET_KEY='{secret_key}' STATIC_ROOT=/mnt/volumes/statics/static/ MEDIA_ROOT=/mnt/volumes/statics/uploaded/ +ASSETS_ROOT=/mnt/volumes/statics/assets/ GEOIP_PATH=/mnt/volumes/statics/geoip.db CACHE_BUSTING_STATIC_ENABLED=False diff --git a/geonode/assets/__init__.py b/geonode/assets/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/assets/apps.py b/geonode/assets/apps.py new file mode 100644 index 00000000000..c994f7bdb09 --- /dev/null +++ b/geonode/assets/apps.py @@ -0,0 +1,35 @@ +######################################################################### +# +# Copyright (C) 2016 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from django.apps import AppConfig + +from geonode.notifications_helper import NotificationsAppConfigBase + + +class BaseAppConfig(NotificationsAppConfigBase, AppConfig): + name = "geonode.assets" + + def ready(self): + super().ready() + run_setup_hooks() + + +def run_setup_hooks(*args, **kwargs): + from geonode.assets.handlers import asset_handler_registry + + asset_handler_registry.init_registry() diff --git a/geonode/assets/handlers.py b/geonode/assets/handlers.py new file mode 100644 index 00000000000..ad2190b46ce --- /dev/null +++ b/geonode/assets/handlers.py @@ -0,0 +1,91 @@ +import logging + +from django.conf import settings +from django.http import HttpResponse +from django.utils.module_loading import import_string + +from geonode.assets.models import Asset + +logger = logging.getLogger(__name__) + + +class AssetHandlerInterface: + + def handled_asset_class(self): + raise NotImplementedError() + + def create(self, title, description, type, owner, *args, **kwargs): + raise NotImplementedError() + + def remove_data(self, asset: Asset, **kwargs): + raise NotImplementedError() + + def replace_data(self, asset: Asset, files: list): + raise NotImplementedError() + + def clone(self, asset: Asset) -> Asset: + """ + Creates a copy in the DB and copies the underlying data as well + """ + raise NotImplementedError() + + def create_link_url(self, asset: Asset) -> str: + raise NotImplementedError() + + def get_download_handler(self, asset: Asset): + raise NotImplementedError() + + def get_storage_manager(self, asset): + raise NotImplementedError() + + +class AssetDownloadHandlerInterface: + + def create_response(self, asset: Asset, attachment: bool = False, basename=None) -> HttpResponse: + raise NotImplementedError() + + +class AssetHandlerRegistry: + _registry = {} + _default_handler = None + + def init_registry(self): + self.register_asset_handlers() + self.set_default_handler() + + def register_asset_handlers(self): + for module_path in settings.ASSET_HANDLERS: + handler = import_string(module_path) + self.register(handler) + logger.info(f"Registered Asset handlers: {', '.join(settings.ASSET_HANDLERS)}") + + def set_default_handler(self): + # check if declared class is registered + for handler in self._registry.values(): + if ".".join([handler.__class__.__module__, handler.__class__.__name__]) == settings.DEFAULT_ASSET_HANDLER: + self._default_handler = handler + break + + if self._default_handler is None: + logger.error(f"Could not set default asset handler class {settings.DEFAULT_ASSET_HANDLER}") + else: + logger.info(f"Default Asset handler {settings.DEFAULT_ASSET_HANDLER}") + + def register(self, asset_handler_class): + self._registry[asset_handler_class.handled_asset_class()] = asset_handler_class() + + def get_default_handler(self) -> AssetHandlerInterface: + return self._default_handler + + def get_handler(self, asset): + asset_cls = asset if isinstance(asset, type) else asset.__class__ + ret = self._registry.get(asset_cls, None) + if not ret: + logger.warning(f"Could not find asset handler for {asset_cls}::{asset.__class__}") + logger.warning("Available asset types:") + for k, v in self._registry.items(): + logger.warning(f"{k} --> {v.__class__.__name__}") + return ret + + +asset_handler_registry = AssetHandlerRegistry() diff --git a/geonode/assets/local.py b/geonode/assets/local.py new file mode 100644 index 00000000000..26f6937f550 --- /dev/null +++ b/geonode/assets/local.py @@ -0,0 +1,159 @@ +import datetime +import logging +import os + +from django.conf import settings +from django.http import HttpResponse +from django.urls import reverse +from django_downloadview import DownloadResponse + +from geonode.assets.handlers import asset_handler_registry, AssetHandlerInterface, AssetDownloadHandlerInterface +from geonode.assets.models import LocalAsset +from geonode.storage.manager import DefaultStorageManager, StorageManager +from geonode.utils import build_absolute_uri, mkdtemp + +logger = logging.getLogger(__name__) + +_asset_storage_manager = StorageManager( + concrete_storage_manager=DefaultStorageManager(location=os.path.dirname(settings.ASSETS_ROOT)) +) + + +class LocalAssetHandler(AssetHandlerInterface): + @staticmethod + def handled_asset_class(): + return LocalAsset + + def get_download_handler(self, asset): + return LocalAssetDownloadHandler() + + def get_storage_manager(self, asset): + return _asset_storage_manager + + def _create_asset_dir(self): + return os.path.normpath( + mkdtemp(dir=settings.ASSETS_ROOT, prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S")) + ) + + def create(self, title, description, type, owner, files=None, clone_files=False, *args, **kwargs): + if not files: + raise ValueError("File(s) expected") + + if clone_files: + prefix = datetime.datetime.now().strftime("%Y%m%d%H%M%S") + files = _asset_storage_manager.copy_files_list(files, dir=settings.ASSETS_ROOT, dir_prefix=prefix) + # TODO: please note the copy_files_list will make flat any directory structure + + asset = LocalAsset( + title=title, + description=description, + type=type, + owner=owner, + created=datetime.datetime.now(), + location=files, + ) + asset.save() + return asset + + def remove_data(self, asset: LocalAsset): + """ + Removes the files related to an Asset. + Only files within the Assets directory are removed + """ + removed_dir = set() + for file in asset.location: + is_managed = self._is_file_managed(file) + if is_managed: + logger.info(f"Removing asset file {file}") + _asset_storage_manager.delete(file) + removed_dir.add(os.path.dirname(file)) + else: + logger.info(f"Not removing asset file outside asset directory {file}") + + # TODO: in case of subdirs, make sure that all the tree is removed in the proper order + for dir in removed_dir: + if not os.path.exists(dir): + logger.warning(f"Trying to remove not existing asset directory {dir}") + continue + if not os.listdir(dir): + logger.info(f"Removing empty asset directory {dir}") + os.rmdir(dir) + + def replace_data(self, asset: LocalAsset, files: list): + self.remove_data(asset) + asset.location = files + asset.save() + + def clone(self, source: LocalAsset) -> LocalAsset: + # get a new asset instance to be edited and stored back + asset = LocalAsset.objects.get(pk=source.pk) + # only copy files if they are managed + if self._are_files_managed(asset.location): + asset.location = _asset_storage_manager.copy_files_list( + asset.location, dir=settings.ASSETS_ROOT, dir_prefix=datetime.datetime.now().strftime("%Y%m%d%H%M%S") + ) + # it's a polymorphic object, we need to null both IDs + # https://django-polymorphic.readthedocs.io/en/stable/advanced.html#copying-polymorphic-objects + asset.pk = None + asset.id = None + asset.save() + asset.refresh_from_db() + return asset + + def create_download_url(self, asset) -> str: + return build_absolute_uri(reverse("assets-download", args=(asset.pk,))) + + def create_link_url(self, asset) -> str: + return build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + + def _is_file_managed(self, file) -> bool: + assets_root = os.path.normpath(settings.ASSETS_ROOT) + return file.startswith(assets_root) + + def _are_files_managed(self, files: list) -> bool: + """ + :param files: files to be checked + :return: True if all files are managed, False is no file is managed + :raise: ValueError if both managed and unmanaged files are in the list + """ + managed = unmanaged = None + for file in files: + if self._is_file_managed(file): + managed = True + else: + unmanaged = True + if managed and unmanaged: + logger.error(f"Both managed and unmanaged files are present: {files}") + raise ValueError("Both managed and unmanaged files are present") + + return bool(managed) + + +class LocalAssetDownloadHandler(AssetDownloadHandlerInterface): + + def create_response(self, asset: LocalAsset, attachment: bool = False, basename=None) -> HttpResponse: + if not asset.location: + return HttpResponse("Asset does not contain any data", status=500) + + if len(asset.location) > 1: + logger.warning("TODO: Asset contains more than one file. Download needs to be implemented") + + file0 = asset.location[0] + filename = os.path.basename(file0) + orig_base, ext = os.path.splitext(filename) + outname = f"{basename or orig_base}{ext}" + + if _asset_storage_manager.exists(file0): + logger.info(f"Returning file {file0} with name {outname}") + + return DownloadResponse( + _asset_storage_manager.open(file0).file, + basename=f"{outname}", + attachment=attachment, + ) + else: + logger.warning(f"Internal file {file0} not found for asset {asset.id}") + return HttpResponse(f"Internal file not found for asset {asset.id}", status=500) + + +asset_handler_registry.register(LocalAssetHandler) diff --git a/geonode/assets/migrations/0001_initial.py b/geonode/assets/migrations/0001_initial.py new file mode 100644 index 00000000000..8a1ef0849a4 --- /dev/null +++ b/geonode/assets/migrations/0001_initial.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2.9 on 2024-04-24 10:02 + +from django.conf import settings +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("base", "0091_alter_hierarchicalkeyword_slug"), + ] + + operations = [ + migrations.CreateModel( + name="Asset", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("title", models.CharField(max_length=255)), + ("description", models.TextField(blank=True, null=True)), + ("type", models.CharField(max_length=255)), + ("created", models.DateTimeField(auto_now_add=True)), + ("owner", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "polymorphic_ctype", + models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="polymorphic_%(app_label)s.%(class)s_set+", + to="contenttypes.contenttype", + ), + ), + ], + options={ + "verbose_name_plural": "Assets", + }, + ), + migrations.CreateModel( + name="LocalAsset", + fields=[ + ( + "asset_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="assets.asset", + ), + ), + ("location", models.JSONField(blank=True, default=list)), + ], + options={ + "verbose_name_plural": "Local assets", + }, + bases=("assets.asset",), + ), + ] diff --git a/geonode/assets/migrations/__init__.py b/geonode/assets/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/assets/models.py b/geonode/assets/models.py new file mode 100644 index 00000000000..fe360258209 --- /dev/null +++ b/geonode/assets/models.py @@ -0,0 +1,48 @@ +from django.db import models +from polymorphic.managers import PolymorphicManager +from polymorphic.models import PolymorphicModel +from django.db.models import signals +from django.contrib.auth import get_user_model + + +class Asset(PolymorphicModel): + """ + A generic data linked to a ResourceBase + """ + + title = models.CharField(max_length=255, null=False, blank=False) + description = models.TextField(null=True, blank=True) + type = models.CharField(max_length=255, null=False, blank=False) + owner = models.ForeignKey(get_user_model(), null=False, blank=False, on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + + objects = PolymorphicManager() + + class Meta: + verbose_name_plural = "Assets" + + def __str__(self) -> str: + return super().__str__() + + +class LocalAsset(Asset): + """ + Local resource, will replace the files + """ + + location = models.JSONField(default=list, blank=True) + + class Meta: + verbose_name_plural = "Local assets" + + def __str__(self) -> str: + return super().__str__() + + +def cleanup_asset_data(instance, *args, **kwargs): + from geonode.assets.handlers import asset_handler_registry + + asset_handler_registry.get_handler(instance).remove_data(instance) + + +signals.post_delete.connect(cleanup_asset_data, sender=LocalAsset) diff --git a/geonode/assets/serializers.py b/geonode/assets/serializers.py new file mode 100644 index 00000000000..999cf8005e7 --- /dev/null +++ b/geonode/assets/serializers.py @@ -0,0 +1,78 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +import logging + +from django.contrib.auth import get_user_model + +from dynamic_rest.serializers import DynamicModelSerializer +from dynamic_rest.fields.fields import DynamicComputedField + +from geonode.assets.models import ( + Asset, + LocalAsset, +) + +logger = logging.getLogger(__name__) + + +class ClassTypeField(DynamicComputedField): + + def get_attribute(self, instance): + return type(instance).__name__ + + +class SimpleUserSerializer(DynamicModelSerializer): + class Meta: + model = get_user_model() + name = "user" + fields = ("pk", "username") + + +class AssetSubclassField(DynamicComputedField): + """ + Just an ugly hack. + TODO: We need a way to automatically use a proper serializer for each Asset subclass + in order to render different instances in a list + """ + + def get_attribute(self, instance): + if type(instance).__name__ == "LocalAsset": + return {"locations": instance.location} + + return None + + +class AssetSerializer(DynamicModelSerializer): + + owner = SimpleUserSerializer(embed=False) + asset_type = ClassTypeField() + subinfo = AssetSubclassField() + + class Meta: + model = Asset + name = "asset" + # fields = ("pk", "title", "description", "type", "owner", "created") + fields = ("pk", "title", "description", "type", "owner", "created", "asset_type", "subinfo") + + +class LocalAssetSerializer(AssetSerializer): + class Meta(AssetSerializer.Meta): + model = LocalAsset + name = "local_asset" + fields = AssetSerializer.Meta.fields + ("location",) diff --git a/geonode/assets/tests.py b/geonode/assets/tests.py new file mode 100644 index 00000000000..5d94f24dcd3 --- /dev/null +++ b/geonode/assets/tests.py @@ -0,0 +1,199 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import os +import logging + +from django.conf import settings +from django.contrib.auth import get_user_model + +from rest_framework.test import APITestCase + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.local import LocalAssetHandler +from geonode.assets.models import Asset, LocalAsset + +logger = logging.getLogger(__name__) + +TEST_GIF = os.path.join(os.path.dirname(os.path.dirname(__file__)), "base/tests/data/img.gif") + + +class AssetsTests(APITestCase): + + def test_handler_registry(self): + # Test registry + self.assertIsNotNone(asset_handler_registry) + # Test default handler + asset_handler = asset_handler_registry.get_default_handler() + self.assertIsNotNone(asset_handler) + self.assertIsInstance(asset_handler, LocalAssetHandler, "Bad default Asset handler found") + # Test None + self.assertIsNone(asset_handler_registry.get_handler(None)) + # Test class without handler + self.assertIsNone(asset_handler_registry.get_handler(AssetsTests)) + + def test_creation_and_delete_data_cloned(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + assets_root = os.path.normpath(settings.ASSETS_ROOT) + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[TEST_GIF], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + self.assertIsNotNone(reloaded) + self.assertIsInstance(reloaded, LocalAsset) + file = reloaded.location[0] + self.assertTrue(os.path.exists(file), "Asset file does not exist") + self.assertTrue(file.startswith(assets_root), f"Asset file is not inside the assets root: {file}") + + cloned_file = file + reloaded.delete() + self.assertFalse(Asset.objects.filter(pk=asset.pk).exists()) + self.assertFalse(os.path.exists(cloned_file)) + self.assertFalse(os.path.exists(os.path.dirname(cloned_file))) + self.assertTrue(os.path.exists(TEST_GIF)) + + def test_creation_and_delete_data_external(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[TEST_GIF], + clone_files=False, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + self.assertIsNotNone(reloaded) + self.assertIsInstance(reloaded, LocalAsset) + file = reloaded.location[0] + self.assertEqual(TEST_GIF, file) + + reloaded.delete() + self.assertFalse(Asset.objects.filter(pk=asset.pk).exists()) + self.assertTrue(os.path.exists(TEST_GIF)) + + def test_clone_and_delete_data_managed(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[TEST_GIF], + clone_files=True, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + cloned = asset_handler.clone(reloaded) + self.assertNotEqual(reloaded.pk, cloned.pk) + + reloaded_file = reloaded.location[0] + cloned_file = cloned.location[0] + + self.assertNotEqual(reloaded_file, cloned_file) + self.assertTrue(os.path.exists(reloaded_file)) + self.assertTrue(os.path.exists(cloned_file)) + + reloaded.delete() + self.assertFalse(os.path.exists(reloaded_file)) + self.assertTrue(os.path.exists(cloned_file)) + + cloned.delete() + self.assertFalse(os.path.exists(cloned_file)) + + def test_clone_and_delete_data_unmanaged(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[TEST_GIF], + clone_files=False, + ) + asset.save() + self.assertIsInstance(asset, LocalAsset) + + reloaded = Asset.objects.get(pk=asset.pk) + cloned = asset_handler.clone(reloaded) + + self.assertEqual(reloaded.location[0], cloned.location[0]) + self.assertTrue(os.path.exists(reloaded.location[0])) + + reloaded.delete() + self.assertTrue(os.path.exists(reloaded.location[0])) + + cloned.delete() + self.assertTrue(os.path.exists(reloaded.location[0])) + + def test_clone_mixed_data(self): + u, _ = get_user_model().objects.get_or_create(username="admin") + + asset_handler = asset_handler_registry.get_default_handler() + managed_asset = asset_handler.create( + title="Test Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[TEST_GIF], + clone_files=True, + ) + managed_asset.save() + + # TODO: dunno if mixed files should be allowed at all + mixed_asset = asset_handler.create( + title="Mixed Asset", + description="Description of test asset", + type="NeverMind", + owner=u, + files=[TEST_GIF, managed_asset.location[0]], + clone_files=False, # let's keep both managed and unmanaged together + ) + + reloaded = Asset.objects.get(pk=mixed_asset.pk) + + try: + asset_handler.clone(reloaded) + self.fail("A mixed LocalAsset has been cloned") + except ValueError: + pass + + mixed_asset.delete() + managed_asset.delete() diff --git a/geonode/assets/urls.py b/geonode/assets/urls.py new file mode 100644 index 00000000000..956854d2523 --- /dev/null +++ b/geonode/assets/urls.py @@ -0,0 +1,25 @@ +######################################################################### +# +# Copyright (C) 2020 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from geonode.api.urls import router + +from geonode.assets import views + +router.register(r"assets", views.AssetViewSet, "assets") + +urlpatterns = [] diff --git a/geonode/assets/utils.py b/geonode/assets/utils.py new file mode 100644 index 00000000000..9aecc4ebd8d --- /dev/null +++ b/geonode/assets/utils.py @@ -0,0 +1,161 @@ +import logging +import os.path + +from django.http import HttpResponse + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.models import Asset +from geonode.base.models import ResourceBase, Link +from geonode.security.utils import get_visible_resources + + +logger = logging.getLogger(__name__) + + +def get_perms_response(request, asset: Asset): + user = request.user + + # quick check + is_admin = user.is_superuser if user and user.is_authenticated else False + if is_admin or user == asset.owner: + logger.debug("Asset: access allowed by user") + return None + + visibile_res = get_visible_resources(queryset=ResourceBase.objects.filter(link__asset=asset), user=request.user) + + logger.warning("TODO: implement permission check") + if visibile_res.exists(): + logger.debug("Asset: access allowed by Resource") + return None + elif user and user.is_authenticated: + return HttpResponse(status=403) + else: + return HttpResponse(status=401) + + +def get_default_asset(resource: ResourceBase, link_type=None) -> Asset or None: + """ + Get the default asset for a ResourceBase. + + In this first implementation we select the first one -- + in the future there may be further flags to identify the preferred one + """ + filters = {"link__resource": resource} + if link_type: + filters["link__link_type"] = link_type + + return Asset.objects.filter(**filters).first() + + +DEFAULT_TYPES = {"image": ["jpg", "jpeg", "gif", "png", "bmp", "svg"]} + + +def find_type(ext): + return next((datatype for datatype, extensions in DEFAULT_TYPES.items() if ext.lower() in extensions), None) + + +def create_link(resource, asset, link_type=None, extension=None, name=None, mime=None, asset_handler=None, **kwargs): + asset_handler = asset_handler or asset_handler_registry.get_handler(asset) + + if not link_type or not extension or not name: + fallback_name, fallback_ext = os.path.splitext(asset.location[0]) if len(asset.location) else (None, None) + if fallback_ext: + fallback_ext = fallback_ext.lstrip(".") + link_type = link_type or find_type(fallback_ext) if fallback_ext else None + + link = Link( + resource=resource, + asset=asset, + url=asset_handler.create_link_url(asset), + extension=extension or fallback_ext or "Unknown", + link_type=link_type or "data", + name=name or fallback_name or asset.title, + mime=mime or "", + ) + link.save() + return link + + +def create_asset_and_link( + resource, + owner, + files: list, + handler=None, + title=None, + description=None, + link_type=None, + extension=None, + asset_type=None, + mime=None, + clone_files: bool = True, +) -> tuple[Asset, Link]: + + asset_handler = handler or asset_handler_registry.get_default_handler() + asset = link = None + try: + default_title, default_ext = os.path.splitext(next(f for f in files)) if len(files) else (None, None) + if default_ext: + default_ext = default_ext.lstrip(".") + link_type = link_type or find_type(default_ext) if default_ext else None + + asset = asset_handler.create( + title=title or default_title or "Unknown", + description=description or asset_type or "Unknown", + type=asset_type or "Unknown", + owner=owner, + files=files, + clone_files=clone_files, + ) + + link = create_link( + resource, + asset, + asset_handler=asset_handler, + link_type=link_type, + extension=extension, + name=title, + mime=mime, + ) + + return asset, link + except Exception as e: + logger.error(f"Error creating Asset for resource {resource}: {e}", exc_info=e) + rollback_asset_and_link(asset, link) + raise Exception(f"Error creating asset: {e}") + + +def create_asset_and_link_dict(resource, values: dict, clone_files=True): + return create_asset_and_link( + resource, + values["owner"], + values["files"], + title=values.pop("data_title", None), + description=values.pop("description", None), + link_type=values.pop("link_type", None), + extension=values.pop("extension", None), + asset_type=values.pop("data_type", None), + clone_files=clone_files, + ) + + +def copy_assets_and_links(resource, target=None) -> list: + assets_and_links = [] + links_with_assets = Link.objects.filter(resource=resource, asset__isnull=False).prefetch_related("asset") + + for link in links_with_assets: + link.asset = asset_handler_registry.get_handler(link.asset).clone(link.asset) + link.pk = None + link.resource = target + link.save() + assets_and_links.append((link.asset, link)) + return assets_and_links + + +def rollback_asset_and_link(asset, link): + try: + if link: + link.delete() + if asset: + asset.delete() # TODO: make sure we are only deleting from DB and not also the stored data + except Exception as e: + logger.error(f"Could not rollback asset[{asset}] and link[{link}]", exc_info=e) diff --git a/geonode/assets/views.py b/geonode/assets/views.py new file mode 100644 index 00000000000..b5a5990c99d --- /dev/null +++ b/geonode/assets/views.py @@ -0,0 +1,112 @@ +######################################################################### +# +# Copyright (C) 2024 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.shortcuts import get_object_or_404 +from dynamic_rest.viewsets import DynamicModelViewSet +from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter + +from oauth2_provider.contrib.rest_framework import OAuth2Authentication + +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.authentication import SessionAuthentication, BasicAuthentication + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.serializers import AssetSerializer +from geonode.assets.utils import get_perms_response +from geonode.assets.models import Asset + +from geonode.base.api.filters import ( + DynamicSearchFilter, +) +from geonode.base.api.pagination import GeoNodeApiPagination +from geonode.base.api.permissions import UserHasPerms + +logger = logging.getLogger(__name__) + + +class AssetViewSet(DynamicModelViewSet): + """ + API endpoint that allows Assets to be viewed or edited. + """ + + authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] + permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms] + filter_backends = [ + DynamicFilterBackend, + DynamicSortingFilter, + DynamicSearchFilter, + # TODO: add filtering by owner / admin + ] + queryset = Asset.objects.all().order_by("-created") + serializer_class = AssetSerializer # TODO: appropriate Serializer should be switched for each Asset instance + pagination_class = GeoNodeApiPagination + + def list(self, request, *args, **kwargs): + """ + Only for lists, allows access to Assets only to owned ones, or to all of them if the user is an admin + """ + queryset = self.filter_queryset(self.get_queryset()) + + user = request.user + is_admin = user.is_superuser if user and user.is_authenticated else False + + if is_admin: + pass + elif user and user.is_authenticated: + queryset = queryset.filter(owner=user) + else: + queryset = queryset.none() + + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + + def _get_file(self, request, pk, attachment: bool): + asset = get_object_or_404(Asset, pk=pk) + if bad_response := get_perms_response(request, asset): + return bad_response + asset_handler = asset_handler_registry.get_handler(asset) + # TODO: register_event(request, EventType.EVENT_DOWNLOAD, asset) + return asset_handler.get_download_handler(asset).create_response(asset, attachment) + + @action( + detail=False, + url_path="(?P\d+)/download", # noqa + # url_name="asset-download", + methods=["get"], + ) + def download(self, request, pk=None, *args, **kwargs): + return self._get_file(request, pk, True) + + @action( + detail=False, + url_path="(?P\d+)/link", # noqa + # url_name="asset-link", + methods=["get"], + ) + def link(self, request, pk=None, *args, **kwargs): + return self._get_file(request, pk, False) diff --git a/geonode/base/api/tests.py b/geonode/base/api/tests.py index be0ae7c1ff0..74e758704b9 100644 --- a/geonode/base/api/tests.py +++ b/geonode/base/api/tests.py @@ -22,11 +22,11 @@ import sys import json import logging +from builtins import Exception from typing import Iterable from django.test import RequestFactory, override_settings import gisdata - from PIL import Image from io import BytesIO from time import sleep @@ -43,8 +43,10 @@ from rest_framework.test import APITestCase from rest_framework.renderers import JSONRenderer from rest_framework.parsers import JSONParser - +from geonode.resource.manager import resource_manager from guardian.shortcuts import get_anonymous_user + +from geonode.assets.utils import create_asset_and_link from geonode.maps.models import Map, MapLayer from geonode.tests.base import GeoNodeBaseTestSupport @@ -2221,16 +2223,22 @@ def test_manager_can_edit_map(self): ) def test_resource_service_copy(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) - resource = Dataset.objects.create( - owner=get_user_model().objects.get(username="admin"), - name="test_copy", - store="geonode_data", - subtype="vector", - alternate="geonode:test_copy", - uuid=str(uuid4()), - files=list(files_as_dict.values()), + resource = resource_manager.create( + str(uuid4()), + Dataset, + defaults={ + "owner": get_user_model().objects.get(username="admin"), + "name": "test_copy", + "store": "geonode_data", + "subtype": "vector", + "alternate": "geonode:test_copy", + }, + ) + + asset, link = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) ) bobby = get_user_model().objects.get(username="bobby") copy_url = reverse("importer_resource_copy", kwargs={"pk": resource.pk}) @@ -2262,22 +2270,29 @@ def test_resource_service_copy(self): cloned_resource = Dataset.objects.last() self.assertEqual(cloned_resource.owner.username, "admin") # clone dataset with invalid file - resource.files = ["/path/invalid_file.wrong"] - resource.save() + # resource.files = ["/path/invalid_file.wrong"] + # resource.save() + asset.location = ["/path/invalid_file.wrong"] + asset.save() response = self.client.put(copy_url) + self.assertEqual(response.status_code, 400) self.assertEqual(response.json()["message"], "Resource can not be cloned.") # clone dataset with no files - resource.files = [] - resource.save() + link.delete() + asset.delete() response = self.client.put(copy_url) + self.assertEqual(response.status_code, 400) self.assertEqual(response.json()["message"], "Resource can not be cloned.") # clean - resource.delete() + try: + resource.delete() + except Exception as e: + logger.warning(f"Can't delete test resource {resource}", exc_info=e) def test_resource_service_copy_with_perms_dataset(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) resource = Dataset.objects.create( owner=get_user_model().objects.get(username="admin"), @@ -2287,7 +2302,9 @@ def test_resource_service_copy_with_perms_dataset(self): alternate="geonode:test_copy", resource_type="dataset", uuid=str(uuid4()), - files=list(files_as_dict.values()), + ) + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) ) self._assertCloningWithPerms(resource) @@ -2295,21 +2312,25 @@ def test_resource_service_copy_with_perms_dataset(self): @override_settings(ASYNC_SIGNALS=False) def test_resource_service_copy_with_perms_dataset_set_default_perms(self): with self.settings(ASYNC_SIGNALS=False): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) - resource = Dataset.objects.create( - owner=get_user_model().objects.get(username="admin"), - name="test_copy_with_perms", - store="geonode_data", - subtype="vector", - alternate="geonode:test_copy_with_perms", - resource_type="dataset", - uuid=str(uuid4()), - files=list(files_as_dict.values()), + resource = resource_manager.create( + None, + resource_type=Dataset, + defaults={ + "owner": get_user_model().objects.first(), + "title": "test_copy_with_perms", + "name": "test_copy_with_perms", + "is_approved": True, + "store": "geonode_data", + "subtype": "vector", + "resource_type": "dataset", + "files": files_as_dict.values(), + }, ) _perms = { "users": {"bobby": ["base.add_resourcebase", "base.download_resourcebase"]}, - "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebae"]}, + "groups": {"anonymous": ["base.view_resourcebase", "base.download_resourcebase"]}, } resource.set_permissions(_perms) # checking that bobby is in the original dataset perms list @@ -2328,11 +2349,11 @@ def test_resource_service_copy_with_perms_dataset_set_default_perms(self): self.assertEqual("finished", self.client.get(response.json().get("status_url")).json().get("status")) _resource = Dataset.objects.filter(title__icontains="test_copy_with_perms").last() self.assertIsNotNone(_resource) - self.assertFalse("bobby" in "bobby" in [x.username for x in _resource.get_all_level_info().get("users", [])]) - self.assertTrue("admin" in "admin" in [x.username for x in _resource.get_all_level_info().get("users", [])]) + self.assertNotIn("bobby", [x.username for x in _resource.get_all_level_info().get("users", [])]) + self.assertIn("admin", [x.username for x in _resource.get_all_level_info().get("users", [])]) def test_resource_service_copy_with_perms_doc(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) resource = Document.objects.create( owner=get_user_model().objects.get(username="admin"), @@ -2340,23 +2361,25 @@ def test_resource_service_copy_with_perms_doc(self): alternate="geonode:test_copy", resource_type="document", uuid=str(uuid4()), - files=list(files_as_dict.values()), ) - + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) self._assertCloningWithPerms(resource) @override_settings(CELERY_TASK_ALWAYS_EAGER=True) def test_resource_service_copy_with_perms_map(self): - files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_water.shp") + files = os.path.join(gisdata.GOOD_DATA, "vector/single_point.shp") files_as_dict, _ = get_files(files) resource = Document.objects.create( owner=get_user_model().objects.get(username="admin"), alternate="geonode:test_copy", resource_type="map", uuid=str(uuid4()), - files=list(files_as_dict.values()), ) - + _, _ = create_asset_and_link( + resource, get_user_model().objects.get(username="admin"), list(files_as_dict.values()) + ) self._assertCloningWithPerms(resource) def _assertCloningWithPerms(self, resource): diff --git a/geonode/base/enumerations.py b/geonode/base/enumerations.py index 4d956e8331f..66430b48bd5 100644 --- a/geonode/base/enumerations.py +++ b/geonode/base/enumerations.py @@ -19,7 +19,7 @@ from django.utils.translation import gettext_lazy as _ -LINK_TYPES = ["original", "data", "image", "metadata", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS"] +LINK_TYPES = ["original", "uploaded", "data", "image", "metadata", "html", "OGC:WMS", "OGC:WFS", "OGC:WCS"] HIERARCHY_LEVELS = ( ("series", _("series")), diff --git a/geonode/base/migrations/0091_create_link_asset_alter_link_type.py b/geonode/base/migrations/0091_create_link_asset_alter_link_type.py new file mode 100644 index 00000000000..e3310ca764d --- /dev/null +++ b/geonode/base/migrations/0091_create_link_asset_alter_link_type.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.9 on 2024-04-24 10:34 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("base", "0091_alter_hierarchicalkeyword_slug"), + ("assets", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="link", + name="asset", + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to="assets.asset"), + ), + migrations.AlterField( + model_name="link", + name="link_type", + field=models.CharField( + choices=[ + ("original", "original"), + ("uploaded", "uploaded"), + ("data", "data"), + ("image", "image"), + ("metadata", "metadata"), + ("html", "html"), + ("OGC:WMS", "OGC:WMS"), + ("OGC:WFS", "OGC:WFS"), + ("OGC:WCS", "OGC:WCS"), + ], + max_length=255, + ), + ), + ] diff --git a/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py b/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py new file mode 100644 index 00000000000..f8077479217 --- /dev/null +++ b/geonode/base/migrations/0092_migrate and_remove_resourcebase_files.py @@ -0,0 +1,95 @@ +# Generated by Django 4.2.9 on 2024-03-12 11:55 +import logging +import os + +from django.db import migrations +from django.db.models import Q +from django.conf import settings +from django.contrib.auth import get_user_model +from django.urls import reverse + +from geonode.base.models import Link +from geonode.assets.models import LocalAsset +from geonode.utils import build_absolute_uri + +logger = logging.getLogger(__name__) + + +def migrate_files(apps, schema_editor): + + def get_ext(filename): + try: + return os.path.splitext(filename)[1][1:] + except Exception as e: + logger.warning(f"Could not find extension for Resource '{res_hm.title}, file '{filename}': {e}") + return None + + ResourceBase_hm = apps.get_model('base', 'ResourceBase') + Dataset_hm = apps.get_model('layers', 'Dataset') + Document_hm = apps.get_model('documents', 'Document') + + if hasattr(ResourceBase_hm, "files"): + # looping on available resources with files to generate the LocalAssets + for res_hm in ResourceBase_hm.objects.exclude(Q(files__isnull=True) | Q(files__exact=[])).iterator(): + # resolving the real owner instance, since resource.owner is an historical model and cant be used directly + owner = get_user_model().objects.get(pk=res_hm.owner.id) + # logger.warning(f"Creating ASSET for {resource.id} -- owner:{type(resource.owner)} --> {resource.owner}") + + files = res_hm.files + # creating the local asset object + asset = LocalAsset( + title="Files", + description="Original uploaded files", + owner=owner, + location=files + ) + asset.save() + + ### creating the association between asset and Link + + # no existing "uploaded" links exist, so create them right away + # otherwise we create the link with the assigned asset + if dataset_hm := Dataset_hm.objects.filter(pk=res_hm.id).first(): + url = build_absolute_uri(reverse("assets-download", args=(asset.pk,))) + elif doc_hm := Document_hm.objects.filter(pk=res_hm.id).first(): + url = build_absolute_uri(reverse("assets-link", args=(asset.pk,))) + else: + raise TypeError(f'ResourceBase {res_hm.id}::"{res_hm.title} has unhandled type"') + + if len(files) == 1: + ext = get_ext(files[0]) + else: + ext = None + for file in files: + for filetype in settings.SUPPORTED_DATASET_FILE_TYPES: + file_ext = get_ext(file) + if file_ext in filetype["ext"]: + ext = filetype["id"] + break + if ext: + break + + Link.objects.create( + resource_id=res_hm.id, + asset=asset, + link_type="uploaded", + name="Original upload", + extension=ext or "unknown", + url=url + ) + + +class Migration(migrations.Migration): + + dependencies = [ + + ("base", "0091_create_link_asset_alter_link_type"), + ] + + operations = [ + migrations.RunPython(migrate_files, migrations.RunPython.noop), + migrations.RemoveField( + model_name="resourcebase", + name="files", + ), + ] diff --git a/geonode/base/models.py b/geonode/base/models.py index 0c83f340898..dd440b435c4 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -88,7 +88,6 @@ from urllib.parse import urlsplit, urljoin from geonode.storage.manager import storage_manager - logger = logging.getLogger(__name__) @@ -607,26 +606,13 @@ def upload_files(resource_id, files, force=False): @staticmethod def cleanup_uploaded_files(resource_id): """Remove uploaded files, if any""" + from geonode.assets.utils import get_default_asset + if ResourceBase.objects.filter(id=resource_id).exists(): _resource = ResourceBase.objects.filter(id=resource_id).get() - _uploaded_folder = None - if _resource.files: - for _file in _resource.files: - try: - if storage_manager.exists(_file): - if not _uploaded_folder: - _uploaded_folder = os.path.split(storage_manager.path(_file))[0] - storage_manager.delete(_file) - except Exception as e: - logger.warning(e) - try: - if _uploaded_folder and storage_manager.exists(_uploaded_folder): - storage_manager.delete(_uploaded_folder) - except Exception as e: - logger.warning(e) - - # Do we want to delete the files also from the resource? - ResourceBase.objects.filter(id=resource_id).update(files={}) + asset = get_default_asset(_resource) # TODO: make sure to select the proper "uploaded" asset + if asset: + asset.delete() # Remove generated thumbnails, if any filename = f"{_resource.get_real_instance().resource_type}-{_resource.get_real_instance().uuid}" @@ -904,8 +890,6 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): _("Metadata"), default=False, help_text=_("If true, will be excluded from search") ) - files = JSONField(null=True, default=list, blank=True) - blob = JSONField(null=True, default=dict, blank=True) subtype = models.CharField(max_length=128, null=True, blank=True) @@ -1086,6 +1070,11 @@ def delete(self, notify=True, *args, **kwargs): resource_manager.remove_permissions(self.uuid, instance=self.get_real_instance()) + # delete assets. TODO: when standalone Assets will be allowed, only dependable Assets shall be removed + links_with_assets = Link.objects.filter(resource=self, asset__isnull=False).prefetch_related("asset") + for link in links_with_assets: + link.asset.delete() + if hasattr(self, "class_name") and notify: notice_type_label = f"{self.class_name.lower()}_deleted" recipients = get_notification_recipients(notice_type_label, resource=self) @@ -1278,10 +1267,12 @@ def instance_is_processed(self): @property def is_copyable(self): - from geonode.geoserver.helpers import select_relevant_files - if self.resource_type == "dataset": - allowed_file = select_relevant_files(get_allowed_extensions(), self.files) + from geonode.assets.utils import get_default_asset + from geonode.geoserver.helpers import select_relevant_files + + asset = get_default_asset(self) # TODO: maybe we need to filter by original files + allowed_file = select_relevant_files(get_allowed_extensions(), asset.location) if asset else [] return len(allowed_file) != 0 return True @@ -2024,6 +2015,7 @@ class Link(models.Model): name = models.CharField(max_length=255, help_text=_('For example "View in Google Earth"')) mime = models.CharField(max_length=255, help_text=_('For example "text/xml"')) url = models.TextField(max_length=1000) + asset = models.ForeignKey("assets.Asset", null=True, on_delete=models.CASCADE) objects = LinkManager() diff --git a/geonode/base/populate_test_data.py b/geonode/base/populate_test_data.py index faa8c3b91f6..d8cc78afd18 100644 --- a/geonode/base/populate_test_data.py +++ b/geonode/base/populate_test_data.py @@ -26,7 +26,6 @@ from taggit.models import TaggedItem from datetime import datetime, timedelta -from django.conf import settings from django.db import transaction from django.utils import timezone from django.db.utils import IntegrityError @@ -36,6 +35,7 @@ from django.contrib.auth.models import Permission, Group from django.core.files.uploadedfile import SimpleUploadedFile +from geonode.assets.utils import create_asset_and_link from geonode.maps.models import Map from geonode.base import enumerations from geonode.layers.models import Dataset @@ -54,7 +54,7 @@ b"GIF87a\x01\x00\x01\x00\x80\x01\x00\x00\x00\x00ccc,\x00" b"\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" ) f = SimpleUploadedFile("test_img_file.gif", imgfile.read(), "image/gif") -dfile = [f"{settings.MEDIA_ROOT}/img.gif"] +dfile = [f"{os.path.dirname(__file__)}/tests/data/img.gif"] def all_public(): @@ -268,11 +268,12 @@ def create_models(type=None, integration=False): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid="EPSG:4326", - files=dfile, + # files=dfile, extension="gif", metadata_only=title == "doc metadata true", ), ) + _, _ = create_asset_and_link(m, m.owner, dfile) m.set_default_permissions(owner=user) m.clear_dirty_state() m.set_processing_state(enumerations.STATE_PROCESSED) @@ -472,11 +473,11 @@ def create_single_doc(name, owner=None, **kwargs): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid="EPSG:4326", - files=dfile, resource_type="document", **kwargs, ), ) + _, _ = create_asset_and_link(m, m.owner, dfile) m.set_default_permissions(owner=owner or admin) m.clear_dirty_state() m.set_processing_state(enumerations.STATE_PROCESSED) @@ -504,10 +505,10 @@ def create_single_geoapp(name, resource_type="geostory", owner=None, **kwargs): bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), ll_bbox_polygon=Polygon.from_bbox((bbox_x0, bbox_y0, bbox_x1, bbox_y1)), srid="EPSG:4326", - files=dfile, **kwargs, ), ) + _, _ = create_asset_and_link(m, m.owner, dfile) m.set_default_permissions(owner=owner or admin) m.clear_dirty_state() m.set_processing_state(enumerations.STATE_PROCESSED) diff --git a/geonode/base/tests/data/img.gif b/geonode/base/tests/data/img.gif new file mode 100644 index 00000000000..56959b6411a Binary files /dev/null and b/geonode/base/tests/data/img.gif differ diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index d8fe1ca2395..d68ca3a09b8 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -28,6 +28,7 @@ from oauth2_provider.contrib.rest_framework import OAuth2Authentication from geonode import settings +from geonode.assets.utils import create_asset_and_link from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter from geonode.base.api.mixins import AdvertisedListMixin from geonode.base.api.pagination import GeoNodeApiPagination @@ -47,6 +48,7 @@ import logging + logger = logging.getLogger(__name__) @@ -119,16 +121,20 @@ def perform_create(self, serializer): "extension": extension, "resource_type": "document", } - if file: - manager = StorageManager(remote_files={"base_file": file}) - manager.clone_remote_files() - payload["files"] = [manager.get_retrieved_paths().get("base_file")] if doc_url: payload["doc_url"] = doc_url payload["sourcetype"] = enumerations.SOURCE_TYPE_REMOTE resource = serializer.save(**payload) + if file: + manager = StorageManager(remote_files={"base_file": file}) + manager.clone_remote_files() + create_asset_and_link( + resource, self.request.user, [manager.get_retrieved_paths().get("base_file")], clone_files=True + ) + manager.delete_retrieved_paths(force=True) + resource.set_missing_info() resourcebase_post_save(resource.get_real_instance()) resource_manager.set_permissions(None, instance=resource, permissions=None, created=True) @@ -136,6 +142,7 @@ def perform_create(self, serializer): resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=False) return resource except Exception as e: + logger.error(f"Error creating document {serializer.validated_data}", exc_info=e) if manager: manager.delete_retrieved_paths() raise e diff --git a/geonode/documents/models.py b/geonode/documents/models.py index 25cc5c2b86d..cdd069929a5 100644 --- a/geonode/documents/models.py +++ b/geonode/documents/models.py @@ -26,6 +26,7 @@ from django.utils.functional import classproperty from django.utils.translation import gettext_lazy as _ +from geonode.assets.models import Asset from geonode.client.hooks import hookset from geonode.base.models import ResourceBase from geonode.groups.conf import settings as groups_settings @@ -76,6 +77,11 @@ def compact_permission_labels(cls): "owner": _("Owner"), } + @property + def files(self): + asset = Asset.objects.filter(link__resource=self).first() + return asset.location if asset else [] + @property def name(self): if not self.title: diff --git a/geonode/documents/tasks.py b/geonode/documents/tasks.py index e0ed9617354..316bdafdfe8 100644 --- a/geonode/documents/tasks.py +++ b/geonode/documents/tasks.py @@ -26,6 +26,8 @@ from geonode.celery_app import app from geonode.storage.manager import StorageManager +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset from ..base.models import ResourceBase from .models import Document @@ -90,7 +92,7 @@ def create_document_thumbnail(self, object_id): """ logger.debug(f"Generating thumbnail for document #{object_id}.") - storage_manager = StorageManager() + default_storage_manager = StorageManager() try: document = Document.objects.get(id=object_id) @@ -104,15 +106,24 @@ def create_document_thumbnail(self, object_id): centering = (0.5, 0.5) doc_path = None - if document.files: - doc_path = storage_manager.path(document.files[0]) + + # get asset of the resource + asset = get_default_asset(document) + if not asset and not document.doc_url: + raise Exception("Document has neither an associated Asset nor a link, cannot generate thumbnail") + + if asset: + handler = asset_handler_registry.get_handler(asset) + asset_storage_manager = handler.get_storage_manager(asset) + doc_path = asset_storage_manager.path(asset.location[0]) elif document.doc_url: doc_path = document.doc_url remove_tmp_file = True + asset_storage_manager = default_storage_manager if document.is_image: try: - image_file = storage_manager.open(doc_path) + image_file = asset_storage_manager.open(doc_path) except Exception as e: logger.debug(f"Could not generate thumbnail from remote document {document.doc_url}: {e}") @@ -129,12 +140,12 @@ def create_document_thumbnail(self, object_id): if image_file is not None: image_file.close() if remove_tmp_file: - storage_manager.delete(doc_path) + default_storage_manager.delete(doc_path) elif doc_renderer.supports(doc_path): # in case it's a remote document we want to retrieve it first if document.doc_url: - doc_path = storage_manager.open(doc_path).name + doc_path = default_storage_manager.open(doc_path).name remove_tmp_file = True try: thumbnail_content = doc_renderer.render(doc_path) @@ -145,7 +156,7 @@ def create_document_thumbnail(self, object_id): print(e) finally: if remove_tmp_file: - storage_manager.delete(doc_path) + default_storage_manager.delete(doc_path) if not thumbnail_content: logger.warning(f"Thumbnail for document #{object_id} empty.") ResourceBase.objects.filter(id=document.id).update(thumbnail_url=None) diff --git a/geonode/documents/tests.py b/geonode/documents/tests.py index e8adaa83c6e..69a82b78903 100644 --- a/geonode/documents/tests.py +++ b/geonode/documents/tests.py @@ -42,6 +42,8 @@ from guardian.shortcuts import get_anonymous_user +from geonode.assets.utils import create_asset_and_link +from geonode.base.forms import LinkedResourceForm from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.compat import ensure_string @@ -58,7 +60,9 @@ from geonode.upload.api.exceptions import FileUploadLimitException from .forms import DocumentCreateForm -from ..base.forms import LinkedResourceForm + + +TEST_GIF = os.path.join(os.path.dirname(__file__), "tests/data/img.gif") class DocumentsTest(GeoNodeBaseTestSupport): @@ -113,10 +117,10 @@ def test_document_mimetypes_rendering(self): def test_create_document_with_no_rel(self, thumb): """Tests the creation of a document with no relations""" thumb.return_value = True - f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) - c = Document.objects.create(files=f, owner=superuser, title="theimg") + c = Document.objects.create(owner=superuser, title="theimg") + _, _ = create_asset_and_link(c, superuser, [TEST_GIF]) c.set_default_permissions() self.assertEqual(Document.objects.get(pk=c.id).title, "theimg") @@ -412,11 +416,11 @@ def test_ajax_document_permissions(self, create_thumb): """Verify that the ajax_document_permissions view is behaving as expected""" create_thumb.return_value = True # Setup some document names to work with - f = [f"{settings.MEDIA_ROOT}/img.gif"] - superuser = get_user_model().objects.get(pk=2) document = resource_manager.create( - None, resource_type=Document, defaults=dict(files=f, owner=superuser, title="theimg", is_approved=True) + None, + resource_type=Document, + defaults=dict(files=[TEST_GIF], owner=superuser, title="theimg", is_approved=True), ) document_id = document.id invalid_document_id = 20 @@ -630,10 +634,10 @@ def setUp(self): def test_create_document_with_links(self): """Tests the creation of document links.""" - f = [f"{settings.MEDIA_ROOT}/img.gif"] superuser = get_user_model().objects.get(pk=2) - d = Document.objects.create(files=f, owner=superuser, title="theimg") + d = Document.objects.create(owner=superuser, title="theimg") + _, _ = create_asset_and_link(d, superuser, [TEST_GIF]) self.assertEqual(Document.objects.get(pk=d.id).title, "theimg") @@ -679,11 +683,10 @@ def setUp(self): self.not_admin = get_user_model().objects.create(username="r-lukaku", is_active=True) self.not_admin.set_password("very-secret") self.not_admin.save() - self.files = [f"{settings.MEDIA_ROOT}/img.gif"] self.test_doc = resource_manager.create( None, resource_type=Document, - defaults=dict(files=self.files, owner=self.not_admin, title="test", is_approved=True), + defaults=dict(files=[TEST_GIF], owner=self.not_admin, title="test", is_approved=True), ) self.perm_spec = {"users": {"AnonymousUser": []}} self.doc_link_url = reverse("document_link", args=(self.test_doc.pk,)) @@ -808,7 +811,7 @@ def test_document_link_with_permissions(self): # Access resource with user logged-in self.client.login(username=self.not_admin.username, password="very-secret") response = self.client.get(self.doc_link_url) - self.assertEqual(response.status_code, 404) + self.assertEqual(response.status_code, 200) # test document link with external url doc = resource_manager.create( None, diff --git a/geonode/documents/utils.py b/geonode/documents/utils.py index 63facbbaf09..8f5ec4ad619 100644 --- a/geonode/documents/utils.py +++ b/geonode/documents/utils.py @@ -23,6 +23,9 @@ # Standard Modules import os import logging + +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset from geonode.storage.manager import storage_manager # Django functionality @@ -31,7 +34,6 @@ from django.template import loader from django.utils.translation import gettext_lazy as _ from django.utils.text import slugify -from django_downloadview.response import DownloadResponse # Geonode functionality from geonode.documents.models import Document @@ -78,10 +80,6 @@ def get_download_response(request, docid, attachment=False): register_event(request, EventType.EVENT_DOWNLOAD, document) filename = slugify(os.path.splitext(os.path.basename(document.title))[0]) - if document.files and storage_manager.exists(document.files[0]): - return DownloadResponse( - storage_manager.open(document.files[0]).file, - basename=f"{filename}.{document.extension}", - attachment=attachment, - ) - return HttpResponse("File is not available", status=404) + asset = get_default_asset(document) + asset_handler = asset_handler_registry.get_handler(asset) + return asset_handler.get_download_handler(asset).create_response(asset, attachment, basename=filename) diff --git a/geonode/documents/views.py b/geonode/documents/views.py index 545fd715647..411f80a4bd0 100644 --- a/geonode/documents/views.py +++ b/geonode/documents/views.py @@ -33,8 +33,10 @@ from django.views.generic.edit import CreateView, UpdateView from django.http import HttpResponse, HttpResponseRedirect, Http404 from django.core.exceptions import PermissionDenied, ObjectDoesNotExist -from geonode.base.api.exceptions import geonode_exception_handler +from geonode.assets.handlers import asset_handler_registry +from geonode.assets.utils import get_default_asset +from geonode.base.api.exceptions import geonode_exception_handler from geonode.client.hooks import hookset from geonode.utils import mkdtemp, resolve_object from geonode.base.views import batch_modify @@ -169,12 +171,21 @@ def form_valid(self, form): owner=self.request.user, doc_url=doc_form.pop("doc_url", None), title=doc_form.pop("title", file.name), + description=doc_form.pop("abstract", None), extension=doc_form.pop("extension", None), + link_type="uploaded", # should be in geonode.base.enumerations.LINK_TYPES + data_title=doc_form.pop("title", file.name), + data_type=doc_form.pop("extension", None), files=[storage_path], ), ) - if tempdir != os.path.dirname(storage_path): - shutil.rmtree(tempdir, ignore_errors=True) + + # Removing the temp file + # TODO: creating a file and then cloning it as an Asset may be slow: we may want to + # create the file directly in the asset dir or to move it + logger.info(f"Removing document temp dir {tempdir}") + shutil.rmtree(tempdir, ignore_errors=True) + else: self.object = resource_manager.create( None, @@ -278,11 +289,17 @@ def form_valid(self, form): if file: tempdir = mkdtemp() dirname = os.path.basename(tempdir) - filepath = storage_manager.save(f"{dirname}/{file.name}", file) + filepath = storage_manager.save(os.path.join(dirname, file.name), file) storage_path = storage_manager.path(filepath) self.object = resource_manager.update( - self.object.uuid, instance=self.object, vals=dict(owner=self.request.user, files=[storage_path]) + self.object.uuid, instance=self.object, vals=dict(owner=self.request.user) ) + + # replace data in existing asset + asset = get_default_asset(self.object, link_type="uploaded") + if asset: + asset_handler_registry.get_handler(asset).replace_data(asset, [storage_path]) + if tempdir != os.path.dirname(storage_path): shutil.rmtree(tempdir, ignore_errors=True) diff --git a/geonode/geoserver/manager.py b/geonode/geoserver/manager.py index a88b641b71f..e3cc4a69378 100644 --- a/geonode/geoserver/manager.py +++ b/geonode/geoserver/manager.py @@ -290,7 +290,6 @@ def import_dataset(self, method: str, uuid: str, /, instance: ResourceBase = Non _to_update = { "name": _name, "title": instance.title or _gs_import_session_info.dataset_name, - "files": kwargs.get("files", None), "workspace": _gs_import_session_info.workspace, "alternate": _alternate, "typename": _alternate, diff --git a/geonode/geoserver/tests/test_manager.py b/geonode/geoserver/tests/test_manager.py index 85672984b00..7d2b9ac2155 100644 --- a/geonode/geoserver/tests/test_manager.py +++ b/geonode/geoserver/tests/test_manager.py @@ -19,6 +19,7 @@ import os import base64 import shutil +from django.test import override_settings import gisdata import requests @@ -54,8 +55,14 @@ def tearDown(self) -> None: return super().tearDown() @on_ogc_backend(geoserver.BACKEND_PACKAGE) + @override_settings(ASYNC_SIGNALS=False, FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777, FILE_UPLOAD_PERMISSIONS=0o7777) def test_revise_resource_value_in_append_should_add_expected_rows_in_the_catalog(self): layer = Dataset.objects.get(name=self.sut.name) + gs_layer = self.cat.get_layer("san_andres_y_providencia_water") + if gs_layer is None: + _gs_import_session_info = self.geoserver_manager._execute_resource_import( + layer, list(self.files_as_dict.values()), self.user, action_type="create" + ) _gs_import_session_info = self.geoserver_manager._execute_resource_import( layer, list(self.files_as_dict.values()), self.user, action_type="append" ) diff --git a/geonode/proxy/templatetags/proxy_lib_tags.py b/geonode/proxy/templatetags/proxy_lib_tags.py index 1135d12bd86..5993d7ef67e 100644 --- a/geonode/proxy/templatetags/proxy_lib_tags.py +++ b/geonode/proxy/templatetags/proxy_lib_tags.py @@ -17,6 +17,7 @@ # ######################################################################### +from geonode.assets.utils import get_default_asset from geonode.base.models import ResourceBase import traceback @@ -52,7 +53,10 @@ def original_link_available(context, resourceid, url): dataset_files = [] if isinstance(instance, ResourceBase): try: - for file in instance.files: + asset_obj = get_default_asset(instance) + # Copy all Dataset related files into a temporary folder + files = asset_obj.location if asset_obj else [] + for file in files: dataset_files.append(file) if not storage_manager.exists(file): return False diff --git a/geonode/proxy/tests.py b/geonode/proxy/tests.py index 2f094488ee0..20a0b091b29 100644 --- a/geonode/proxy/tests.py +++ b/geonode/proxy/tests.py @@ -30,6 +30,7 @@ from urllib.parse import urljoin from django.conf import settings +from geonode.assets.utils import create_asset_and_link from geonode.proxy.templatetags.proxy_lib_tags import original_link_available from django.test.client import RequestFactory from django.core.files.uploadedfile import SimpleUploadedFile @@ -308,12 +309,15 @@ def test_download_url_with_existing_files(self, fopen, fexists): fopen.return_value = SimpleUploadedFile("foo_file.shp", b"scc") dataset = Dataset.objects.all().first() - dataset.files = [ + dataset_files = [ "/tmpe1exb9e9/foo_file.dbf", "/tmpe1exb9e9/foo_file.prj", "/tmpe1exb9e9/foo_file.shp", "/tmpe1exb9e9/foo_file.shx", ] + asset, link = create_asset_and_link( + dataset, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False + ) dataset.save() @@ -331,6 +335,9 @@ def test_download_url_with_existing_files(self, fopen, fexists): self.assertEqual("application/zip", response.headers.get("Content-Type")) self.assertEqual('attachment; filename="CA.zip"', response.headers.get("Content-Disposition")) + link.delete() + asset.delete() + @patch("geonode.storage.manager.storage_manager.exists") @patch("geonode.storage.manager.storage_manager.open") @on_ogc_backend(geoserver.BACKEND_PACKAGE) @@ -339,12 +346,15 @@ def test_download_files(self, fopen, fexists): fopen.return_value = SimpleUploadedFile("foo_file.shp", b"scc") dataset = Dataset.objects.all().first() - dataset.files = [ + dataset_files = [ "/tmpe1exb9e9/foo_file.dbf", "/tmpe1exb9e9/foo_file.prj", "/tmpe1exb9e9/foo_file.shp", "/tmpe1exb9e9/foo_file.shx", ] + asset, link = create_asset_and_link( + dataset, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False + ) dataset.save() @@ -368,6 +378,9 @@ def test_download_files(self, fopen, fexists): self.assertIn(".shx", "".join(zip_files)) self.assertIn(".prj", "".join(zip_files)) + link.delete() + asset.delete() + class OWSApiTestCase(GeoNodeBaseTestSupport): def setUp(self): @@ -420,16 +433,23 @@ def test_should_return_true_if_files_are_available(self, fexists): assert upload - self.resource.files = [ + dataset_files = [ "/tmpe1exb9e9/foo_file.dbf", "/tmpe1exb9e9/foo_file.prj", "/tmpe1exb9e9/foo_file.shp", "/tmpe1exb9e9/foo_file.shx", ] + asset, link = create_asset_and_link( + self.resource, get_user_model().objects.get(username="admin"), dataset_files, clone_files=False + ) + self.resource.save() self.resource.refresh_from_db() actual = original_link_available(self.context, self.resource.resourcebase_ptr_id, self.url) self.assertTrue(actual) + + link.delete() + asset.delete() diff --git a/geonode/proxy/views.py b/geonode/proxy/views.py index b7bca6ee5b9..20fe0158ca5 100644 --- a/geonode/proxy/views.py +++ b/geonode/proxy/views.py @@ -55,6 +55,7 @@ from geonode.base import register_event from geonode.base.auth import get_auth_user, get_token_from_auth_header from geonode.geoserver.helpers import ogc_server_settings +from geonode.assets.utils import get_default_asset from .utils import proxy_urls_registry @@ -245,8 +246,9 @@ def download(request, resourceid, sender=Dataset): dataset_files = [] file_list = [] # Store file info to be returned try: - files = instance.resourcebase_ptr.files + asset_obj = get_default_asset(instance) # Copy all Dataset related files into a temporary folder + files = asset_obj.location if asset_obj else [] for file_path in files: if storage_manager.exists(file_path): dataset_files.append(file_path) diff --git a/geonode/resource/manager.py b/geonode/resource/manager.py index d4c94a1bc66..eefab451b4a 100644 --- a/geonode/resource/manager.py +++ b/geonode/resource/manager.py @@ -21,6 +21,7 @@ import copy import typing import logging +import itertools from uuid import uuid1, uuid4 from abc import ABCMeta, abstractmethod @@ -38,6 +39,8 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist, ValidationError, FieldDoesNotExist + +from geonode.base.models import ResourceBase, LinkedResource from geonode.thumbs.thumbnails import _generate_thumbnail_name from geonode.documents.tasks import create_document_thumbnail from geonode.security.permissions import PermSpecCompact, DATA_STYLABLE_RESOURCES_SUBTYPES @@ -45,9 +48,9 @@ from . import settings as rm_settings from .utils import update_resource, resourcebase_post_save +from geonode.assets.utils import create_asset_and_link_dict, rollback_asset_and_link, copy_assets_and_links, create_link from ..base import enumerations -from ..base.models import ResourceBase, LinkedResource from ..security.utils import AdvancedSecurityWorkflowManager from ..layers.metadata import parse_metadata from ..documents.models import Document @@ -313,20 +316,35 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de if resource_type.objects.filter(uuid=uuid).exists(): return resource_type.objects.filter(uuid=uuid).get() uuid = uuid or str(uuid4()) - _resource, _created = resource_type.objects.get_or_create(uuid=uuid, defaults=defaults) + resource_dict = { # TODO: cleanup params and dicts + k: v + for k, v in defaults.items() + if k not in ("data_title", "data_type", "description", "files", "link_type", "extension", "asset") + } + _resource, _created = resource_type.objects.get_or_create(uuid=uuid, defaults=resource_dict) if _resource and _created: _resource.set_processing_state(enumerations.STATE_RUNNING) try: + # if files exist: create an Asset out of them and link it to the Resource + asset, link = (None, None) # safe init in case of exception + if defaults.get("files", None): + logger.debug(f"Found files when creating resource {_resource}: {defaults['files']}") + asset, link = create_asset_and_link_dict(_resource, defaults, clone_files=True) + elif defaults.get("asset", None): + logger.debug(f"Found asset when creating resource {_resource}: {defaults['asset']}") + link = create_link(_resource, **defaults) + with transaction.atomic(): _resource.set_missing_info() _resource = self._concrete_resource_manager.create( - uuid, resource_type=resource_type, defaults=defaults + uuid, resource_type=resource_type, defaults=resource_dict ) _resource.save() resourcebase_post_save(_resource.get_real_instance()) _resource.set_processing_state(enumerations.STATE_PROCESSED) except Exception as e: logger.exception(e) + rollback_asset_and_link(asset, link) # we are not removing the Asset passed in defaults self.delete(_resource.uuid, instance=_resource) raise e return _resource @@ -440,19 +458,19 @@ def ingest( ) -> ResourceBase: instance = None to_update = defaults.copy() - if "files" in to_update: - to_update.pop("files") + to_update_with_files = {**to_update, **{"files": files}} try: with transaction.atomic(): if resource_type == Document: if "name" in to_update: to_update.pop("name") - if files: - to_update["files"] = storage_manager.copy_files_list(files) - instance = self.create(uuid, resource_type=Document, defaults=to_update) + instance = self.create(uuid, resource_type=Document, defaults=to_update_with_files) elif resource_type == Dataset: if files: - instance = self.create(uuid, resource_type=Dataset, defaults=to_update) + instance = self.create(uuid, resource_type=Dataset, defaults=to_update_with_files) + else: + logger.warning(f"Will not create a Dataset without any file. Values: {defaults}") + if instance: instance = self._concrete_resource_manager.ingest( storage_manager.copy_files_list(files), @@ -523,11 +541,15 @@ def copy( _maplayer.pk = _maplayer.id = None _maplayer.map = _resource.get_real_instance() _maplayer.save() + + assets_and_links = copy_assets_and_links(instance, target=_resource) + # we're just merging all the files together: it won't work once we have multiple assets per resource + # TODO: get the files from the proper Asset, or make the _concrete_resource_manager.copy use assets to_update = {} - try: - to_update = storage_manager.copy(_resource).copy() - except Exception as e: - logger.exception(e) + + files = list(itertools.chain.from_iterable([asset.location for asset, _ in assets_and_links])) + if files: + to_update = {"files": files} _resource = self._concrete_resource_manager.copy(instance, uuid=_resource.uuid, defaults=to_update) diff --git a/geonode/resource/tests.py b/geonode/resource/tests.py index 4b25308afbe..a1850861673 100644 --- a/geonode/resource/tests.py +++ b/geonode/resource/tests.py @@ -148,7 +148,15 @@ def _copy_assert_resource(res, title): # copy with documents res = self.rm.ingest( - dt_files, resource_type=Document, defaults={"title": "relief_san_andres", "owner": self.user} + dt_files, + resource_type=Document, + defaults={ + "title": "relief_san_andres", + "owner": self.user, + "extension": "tif", + "data_title": "relief_san_andres", + "data_type": "tif", + }, ) self.assertTrue(isinstance(res, Document)) _copy_assert_resource(res, "Testing Document 2") @@ -157,7 +165,12 @@ def _copy_assert_resource(res, title): res = self.rm.ingest( dt_files, resource_type=Dataset, - defaults={"owner": self.user, "title": "Testing Dataset", "files": dt_files}, + defaults={ + "owner": self.user, + "title": "Testing Dataset", + "data_title": "relief_san_andres", + "data_type": "tif", + }, ) self.assertTrue(isinstance(res, Dataset)) _copy_assert_resource(res, "Testing Dataset 2") diff --git a/geonode/resource/utils.py b/geonode/resource/utils.py index f0e8e2201e5..78e215173e8 100644 --- a/geonode/resource/utils.py +++ b/geonode/resource/utils.py @@ -29,9 +29,11 @@ from django.utils import timezone from django.core.exceptions import FieldDoesNotExist from django.utils.translation import gettext_lazy as _ -from geonode.utils import OGC_Servers_Handler from django.utils.module_loading import import_string +from geonode.assets.utils import get_default_asset +from geonode.utils import OGC_Servers_Handler + from ..base import enumerations from ..base.models import ( ExtraMetadata, @@ -241,10 +243,13 @@ def update_resource( ] to_update.update(defaults) + resource_dict = { # TODO: cleanup params and dicts + k: v for k, v in to_update.items() if k not in ("data_title", "data_type", "description", "files", "link_type") + } try: - instance.get_real_concrete_instance_class().objects.filter(id=instance.id).update(**to_update) + instance.get_real_concrete_instance_class().objects.filter(id=instance.id).update(**resource_dict) except Exception as e: - logger.error(f"{e} - {to_update}") + logger.error(f"{e} - {resource_dict}") raise # Check for "remote services" availability @@ -322,9 +327,9 @@ def get_alternate_name(instance): def document_post_save(instance, *args, **kwargs): instance.csw_type = "document" - - if instance.files: - _, extension = os.path.splitext(os.path.basename(instance.files[0])) + asset = get_default_asset(instance) + if asset: + _, extension = os.path.splitext(os.path.basename(asset.location[0])) instance.extension = extension[1:] doc_type_map = DOCUMENT_TYPE_MAP doc_type_map.update(getattr(settings, "DOCUMENT_TYPE_MAP", {})) @@ -344,7 +349,7 @@ def document_post_save(instance, *args, **kwargs): mime = mime_type_map.get(ext, "text/plain") url = None - if instance.id and instance.files: + if instance.id and asset: name = "Hosted Document" site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL url = f"{site_url}{reverse('document_download', args=(instance.id,))}" @@ -455,8 +460,10 @@ def resourcebase_post_save(instance, *args, **kwargs): if hasattr(instance, "abstract") and not getattr(instance, "abstract", None): instance.abstract = _("No abstract provided") if hasattr(instance, "title") and not getattr(instance, "title", None) or getattr(instance, "title", "") == "": - if isinstance(instance, Document) and instance.files: - instance.title = os.path.basename(instance.files[0]) + asset = get_default_asset(instance) + files = asset.location if asset else [] + if isinstance(instance, Document) and files: + instance.title = os.path.basename(files[0]) if hasattr(instance, "name") and getattr(instance, "name", None): instance.title = instance.name if ( diff --git a/geonode/security/tests.py b/geonode/security/tests.py index 5ae00733cf4..ce9b40a507f 100644 --- a/geonode/security/tests.py +++ b/geonode/security/tests.py @@ -21,9 +21,11 @@ import base64 import logging import uuid +import os import requests import importlib import mock +import gisdata from requests.auth import HTTPBasicAuth from tastypie.test import ResourceTestCaseMixin @@ -40,7 +42,9 @@ from guardian.shortcuts import assign_perm, get_anonymous_user from geonode import geoserver -from geonode.geoserver.helpers import geofence, gf_utils +from geonode.geoserver.helpers import geofence, gf_utils, gs_catalog +from geonode.geoserver.manager import GeoServerResourceManager +from geonode.layers.utils import get_files from geonode.maps.models import Map from geonode.layers.models import Dataset from geonode.documents.models import Document @@ -742,8 +746,19 @@ def test_perm_specs_synchronization(self): @on_ogc_backend(geoserver.BACKEND_PACKAGE) def test_dataset_permissions(self): # Test permissions on a layer + files = os.path.join(gisdata.GOOD_DATA, "vector/san_andres_y_providencia_poi.shp") + files_as_dict, self.tmpdir = get_files(files) + bobby = get_user_model().objects.get(username="bobby") - layer = create_single_dataset("san_andres_y_providencia_poi") + layer = create_single_dataset( + "san_andres_y_providencia_poi", + { + "owner": self.user, + "title": "Testing Dataset", + "data_title": "relief_san_andres", + "data_type": "tif", + }, + ) layer = resource_manager.update( layer.uuid, instance=layer, notify=False, vals=dict(owner=bobby, workspace=settings.DEFAULT_WORKSPACE) ) @@ -774,6 +789,15 @@ def test_dataset_permissions(self): perm_spec = {"users": {"AnonymousUser": []}, "groups": []} layer.set_permissions(perm_spec) + gs_layer = gs_catalog.get_layer("3Asan_andres_y_providencia_poi") + if gs_layer is None: + GeoServerResourceManager()._execute_resource_import( + layer, + list(files_as_dict.values()), + get_user_model().objects.get(username="admin"), + action_type="create", + ) + url = ( f"{settings.GEOSERVER_LOCATION}ows?" "LAYERS=geonode%3Asan_andres_y_providencia_poi&STYLES=" @@ -786,7 +810,8 @@ def test_dataset_permissions(self): # test view_resourcebase permission on anonymous user response = requests.get(url) - self.assertTrue(response.status_code, 404) + self.assertEqual(response.status_code, 200) + self.assertTrue(b"Could not find layer" in response.content) self.assertEqual(response.headers.get("Content-Type"), "application/vnd.ogc.se_xml;charset=UTF-8") # test WMS with authenticated user that has access to the Dataset @@ -796,7 +821,7 @@ def test_dataset_permissions(self): username=settings.OGC_SERVER["default"]["USER"], password=settings.OGC_SERVER["default"]["PASSWORD"] ), ) - self.assertTrue(response.status_code, 200) + self.assertEqual(response.status_code, 200) self.assertEqual(response.headers.get("Content-Type"), "image/png") # test WMS with authenticated user that has no view_resourcebase: diff --git a/geonode/settings.py b/geonode/settings.py index 84333dc7ef2..ad2d2e33a76 100644 --- a/geonode/settings.py +++ b/geonode/settings.py @@ -300,6 +300,11 @@ # Example: "/home/media/media.lawrence.com/apps/" STATIC_ROOT = os.getenv("STATIC_ROOT", os.path.join(PROJECT_ROOT, "static_root")) +# Absolute path to the directory that hold assets files +# This dir should not be made publicly accessible by nginx, since its content may be private +# Using a sibling of MEDIA_ROOT as default +ASSETS_ROOT = os.getenv("ASSETS_ROOT", os.path.join(os.path.dirname(MEDIA_ROOT.rstrip("/")), "assets")) + # Cache Bustin Settings: enable WhiteNoise compression and caching support # ref: http://whitenoise.evans.io/en/stable/django.html#add-compression-and-caching-support CACHE_BUSTING_STATIC_ENABLED = ast.literal_eval(os.environ.get("CACHE_BUSTING_STATIC_ENABLED", "False")) @@ -2365,3 +2370,10 @@ def get_geonode_catalogue_service(): AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS = ast.literal_eval( os.getenv("AUTO_ASSIGN_REGISTERED_MEMBERS_TO_CONTRIBUTORS", "True") ) + +DEFAULT_ASSET_HANDLER = "geonode.assets.local.LocalAssetHandler" +ASSET_HANDLERS = [ + DEFAULT_ASSET_HANDLER, +] +INSTALLED_APPS += ("geonode.assets",) +GEONODE_APPS += ("geonode.assets",) diff --git a/geonode/storage/data_retriever.py b/geonode/storage/data_retriever.py index bc23997a8a2..f7178937228 100644 --- a/geonode/storage/data_retriever.py +++ b/geonode/storage/data_retriever.py @@ -151,10 +151,12 @@ def __init__(self, files, tranfer_at_creation=False): if tranfer_at_creation: self.transfer_remote_files() - def transfer_remote_files(self): + def transfer_remote_files(self, cloning_directory=None, prefix=None, create_tempdir=True): from geonode.utils import mkdtemp - self.temporary_folder = mkdtemp() + self.temporary_folder = cloning_directory or settings.MEDIA_ROOT + if create_tempdir: + self.temporary_folder = mkdtemp(cloning_directory or settings.MEDIA_ROOT, prefix=prefix) for name, data_item_retriever in self.data_items.items(): file_path = data_item_retriever.transfer_remote_file(self.temporary_folder) self.file_paths[name] = Path(file_path) @@ -172,10 +174,12 @@ def transfer_remote_files(self): os.chmod(self.temporary_folder, settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS) return self.file_paths - def get_paths(self, allow_transfer=False): + def get_paths(self, allow_transfer=False, cloning_directory=None, prefix=None, create_tempdir=True): if not self.file_paths: if allow_transfer: - self.transfer_remote_files() + self.transfer_remote_files( + cloning_directory=cloning_directory, prefix=prefix, create_tempdir=create_tempdir + ) else: raise DataRetrieverExcepion(detail="You can't retrieve paths without clone file first!") return self.file_paths.copy() diff --git a/geonode/storage/manager.py b/geonode/storage/manager.py index 2966fc6924e..4d1cbbb0133 100644 --- a/geonode/storage/manager.py +++ b/geonode/storage/manager.py @@ -124,8 +124,8 @@ class StorageManager(StorageManagerInterface): treat as a file_system file """ - def __init__(self, remote_files: Mapping = {}): - self._concrete_storage_manager = self._get_concrete_manager() + def __init__(self, remote_files: Mapping = {}, concrete_storage_manager=None): + self._concrete_storage_manager = concrete_storage_manager or self._get_concrete_manager() self.data_retriever = DataRetriever(remote_files, tranfer_at_creation=False) def _get_concrete_manager(self): @@ -174,18 +174,19 @@ def replace(self, resource, files: Union[list, BinaryIO]): updated_files["files"] = [self.replace_single_file(resource.files[0], files)] return updated_files - def copy(self, resource): - updated_files = {} - if len(resource.files): - updated_files["files"] = self.copy_files_list(resource.files) - return updated_files + def copy(self, resource, target=None): + raise Exception("This is not the copy you're looking for") + # updated_files = {} + # if len(resource.files): + # updated_files["files"] = self.copy_files_list(resource.files) + # return updated_files - def copy_files_list(self, files: List[str]): + def copy_files_list(self, files: List[str], dir=settings.MEDIA_ROOT, dir_prefix=None, dir_suffix=None): from geonode.utils import mkdtemp out = [] random_suffix = f"{uuid1().hex[:8]}" - new_path = mkdtemp() + new_path = mkdtemp(dir=dir, prefix=dir_prefix, suffix=dir_suffix) if settings.FILE_UPLOAD_DIRECTORY_PERMISSIONS is not None: # value is always set by default as None @@ -242,11 +243,13 @@ def replace_single_file(self, old_file: str, new_file: BinaryIO, prefix: str = N def generate_filename(self, filename): return self._concrete_storage_manager.generate_filename(filename) - def clone_remote_files(self) -> Mapping: + def clone_remote_files(self, cloning_directory=None, prefix=None, create_tempdir=True) -> Mapping: """ Using the data retriever object clone the remote path into a local temporary storage """ - return self.data_retriever.get_paths(allow_transfer=True) + return self.data_retriever.get_paths( + allow_transfer=True, cloning_directory=cloning_directory, prefix=prefix, create_tempdir=create_tempdir + ) def get_retrieved_paths(self) -> Mapping: """ @@ -266,8 +269,8 @@ def delete_retrieved_paths(self, force=False) -> None: class DefaultStorageManager(StorageManagerInterface): - def __init__(self): - self._fsm = FileSystemStorage() + def __init__(self, **kwargs): + self._fsm = FileSystemStorage(**kwargs) def _get_concrete_manager(self): return DefaultStorageManager() diff --git a/geonode/storage/tests.py b/geonode/storage/tests.py index 9a96186ab3e..43adc794ffa 100644 --- a/geonode/storage/tests.py +++ b/geonode/storage/tests.py @@ -401,22 +401,6 @@ def test_storage_manager_replace_single_file(self, path, strg): output = self.sut().replace(dataset, new_file) self.assertListEqual([expected], output["files"]) - @override_settings(FILE_UPLOAD_DIRECTORY_PERMISSIONS=0o777) - @override_settings(FILE_UPLOAD_PERMISSIONS=0o777) - def test_storage_manager_copy(self): - """ - Test that the copy works as expected and the permissions are corerct - """ - dataset = create_single_dataset(name="test_copy") - dataset.files = [os.path.join(f"{self.project_root}", "tests/data/test_sld.sld")] - dataset.save() - output = self.sut().copy(dataset) - - self.assertTrue(os.path.exists(output.get("files")[0])) - self.assertEqual(os.stat(os.path.exists(output.get("files")[0])).st_mode, 8592) - os.remove(output.get("files")[0]) - self.assertFalse(os.path.exists(output.get("files")[0])) - class TestDataRetriever(TestCase): @classmethod diff --git a/geonode/upload/api/tests.py b/geonode/upload/api/tests.py index c607799d07d..a35aea3a274 100644 --- a/geonode/upload/api/tests.py +++ b/geonode/upload/api/tests.py @@ -255,8 +255,15 @@ def test_rest_uploads(self): self.assertEqual(len(response.data["uploads"]), 0) logger.debug(response.data) except Exception: - if resp.json().get("errors"): - layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + json = resp.json() + if json.get("errors"): + logger.error(f"Error in upload: {json}") + try: + layer_name = json.get("errors")[0].split("for : ")[1].split(",")[0] + except IndexError as e: + logger.error(f"Could not parse layername from {json.get('errors')}", exc_info=e) + # TODO: make sure the _cleanup_layer will use the proper layer name + self.skipTest("Error with GeoServer") finally: self._cleanup_layer(layer_name) @@ -276,9 +283,16 @@ def test_rest_uploads_non_interactive(self): exec_id = data.get("execution_id", None) _exec = ExecutionRequest.objects.get(exec_id=exec_id) self.assertEqual(_exec.status, "finished") - except Exception: - if resp.json().get("errors"): - layer_name = resp.json().get("errors")[0].split("for : ")[1].split(",")[0] + except Exception as e: + json = resp.json() + logger.warning(f"Error with GeoServer {json}: {e}", exc_info=e) + if json.get("errors"): + try: + layer_name = json.get("errors")[0].split("for : ")[1].split(",")[0] + except IndexError as e: + logger.error(f"Could not parse layername from {json.get('errors')}", exc_info=e) + # TODO: make sure the _cleanup_layer will use the proper layer name + self.skipTest("Error with GeoServer") finally: self._cleanup_layer(layer_name) diff --git a/geonode/urls.py b/geonode/urls.py index ccab35e950f..d3222300a38 100644 --- a/geonode/urls.py +++ b/geonode/urls.py @@ -127,6 +127,7 @@ re_path(r"^api/v2/", include("geonode.management_commands_http.urls")), re_path(r"^api/v2/api-auth/", include("rest_framework.urls", namespace="geonode_rest_framework")), re_path(r"^api/v2/", include("geonode.facets.urls")), + re_path(r"^api/v2/", include("geonode.assets.urls")), re_path(r"", include(api.urls)), ] diff --git a/geonode/utils.py b/geonode/utils.py index c93b61d7cf7..9268feb51da 100755 --- a/geonode/utils.py +++ b/geonode/utils.py @@ -281,13 +281,13 @@ def all(self): return [self[alias] for alias in self] -def mkdtemp(dir=settings.MEDIA_ROOT): +def mkdtemp(dir=settings.MEDIA_ROOT, prefix=None, suffix=None): if not os.path.exists(dir): os.makedirs(dir, exist_ok=True) tempdir = None while not tempdir: try: - tempdir = tempfile.mkdtemp(dir=dir) + tempdir = tempfile.mkdtemp(dir=dir, prefix=prefix, suffix=suffix) if os.path.exists(tempdir) and os.path.isdir(tempdir): if os.listdir(tempdir): raise Exception("Directory is not empty") diff --git a/requirements.txt b/requirements.txt index 09ce69b0e71..40d80bb7616 100644 --- a/requirements.txt +++ b/requirements.txt @@ -87,7 +87,8 @@ geonode-pinax-notifications==6.0.0.2 # GeoNode org maintained apps. # django-geonode-mapstore-client==4.0.5 git+https://github.com/GeoNode/geonode-mapstore-client.git@master#egg=django_geonode_mapstore_client -git+https://github.com/GeoNode/geonode-importer.git@master#egg=geonode-importer +#git+https://github.com/GeoNode/geonode-importer.git@geonode_12124_assets#egg=geonode-importer +git+https://github.com/GeoNode/geonode-importer.git@assets_data_retriever#egg=geonode-importer django-avatar==8.0.0 geonode-oauth-toolkit==2.2.2.2 geonode-user-messages==2.0.2.2 diff --git a/tasks.py b/tasks.py index b19fa5f7973..82233ee642d 100755 --- a/tasks.py +++ b/tasks.py @@ -341,8 +341,9 @@ def statics(ctx): try: static_root = os.environ.get("STATIC_ROOT", "/mnt/volumes/statics/static/") media_root = os.environ.get("MEDIA_ROOT", "/mnt/volumes/statics/uploaded/") + assets_root = os.environ.get("ASSETS_ROOT", "/mnt/volumes/statics/assets/") - ctx.run(f"mkdir -pv {static_root} {media_root}") + ctx.run(f"mkdir -pv {static_root} {media_root} {assets_root}") ctx.run( f"python manage.py collectstatic --noinput --settings={_localsettings()}", pty=True,