[Fixes #12124] Implementation of assets

geonode/base/migrations/0091_asset_remove_resourcebase_files_and_more.py

@@ -0,0 +1,71 @@
+# Generated by Django 4.2.9 on 2024-03-13 09:38
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django.utils.timezone
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("base", "0090_alter_resourcebase_polymorphic_ctype"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Asset",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("title", models.CharField(max_length=255)),
+                ("description", models.TextField(blank=True, null=True)),
+                ("type", models.CharField(max_length=255)),
+                ("created", models.DateTimeField(default=django.utils.timezone.now)),
+                ("owner", models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+                (
+                    "polymorphic_ctype",
+                    models.ForeignKey(
+                        editable=False,
+                        null=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="polymorphic_%(app_label)s.%(class)s_set+",
+                        to="contenttypes.contenttype",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name_plural": "Assets",
+            },
+        ),
+        migrations.AlterField(
+            model_name="hierarchicalkeyword",
+            name="slug",
+            field=models.SlugField(allow_unicode=True, max_length=100, unique=True, verbose_name="slug"),
+        ),
+        migrations.CreateModel(
+            name="LocalAsset",
+            fields=[
+                (
+                    "asset_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="base.asset",
+                    ),
+                ),
+                ("location", models.CharField(max_length=255)),
+            ],
+            options={
+                "verbose_name_plural": "Local assets",
+            },
+            bases=("base.asset",),
+        ),
+        migrations.AddField(
+            model_name="link",
+            name="asset",
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to="base.asset"),
+        ),
+    ]

geonode/base/migrations/0092_generate_assets.py

@@ -0,0 +1,55 @@
+# Generated by Django 4.2.9 on 2024-03-12 11:55
+
+from django.conf import settings
+from django.db import migrations
+from django.db.models import Q
+from geonode.base.enumerations import LINK_TYPES
+
+from geonode.base.models import Link, LocalAsset, ResourceBase
+
+
+
+def migrate_files(apps, schema_editor):
+    if hasattr(ResourceBase, "files"):
+        # looping on available resources with files to generate the LocalAssets
+        for resource in ResourceBase.objects.exclude(Q(files__isnull=True) | Q(files__exact=[])).iterator():
+            # creating the local asset object
+            asset = LocalAsset(
+                title="files",
+                description="Original uploaded files path",
+                owner=resource.owner,
+                location=resource.files
+            )
+            asset.save()
+            # creating the association between asset and Link
+            # lets check if a original link exists
+
+            link = resource.link_set.filter(link_type="original").first()
+            if link:
+                # if exists, we can assign the asset to the link
+                link.asset = asset
+                link.save()
+            else:
+                # otherwise we create the link with the assigned asset
+                Link.objects.create(
+                    resource=resource.get_self_resource(),
+                    asset=asset,
+                    link_type="original",
+                    name="Original",
+                    url=resource.get_real_instance().download_url
+                )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("base", "0091_asset_remove_resourcebase_files_and_more"),
+    ]
+
+    operations = [
+        migrations.RunPython(migrate_files, migrations.RunPython.noop),
+        migrations.RemoveField(
+            model_name="resourcebase",
+            name="files",
+        ),
+    ]

geonode/base/migrations/0093_alter_localasset_location.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.9 on 2024-03-14 10:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("base", "0092_generate_assets"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="localasset",
+            name="location",
+            field=models.JSONField(blank=True, default=list),
+        ),
+    ]

geonode/base/models.py

@@ -50,7 +50,6 @@
 from django.contrib.contenttypes.models import ContentType
 from django.utils.html import strip_tags
 from mptt.models import MPTTModel, TreeForeignKey
-
 from PIL import Image, ImageOps
 
 from polymorphic.models import PolymorphicModel
@@ -86,7 +85,7 @@
 from geonode.people.enumerations import ROLE_VALUES
 
 from urllib.parse import urlsplit, urljoin
-from geonode.storage.manager import storage_manager
+from geonode.storage.manager import storage_manager, StorageManagerFactory
 
 
 logger = logging.getLogger(__name__)
@@ -904,8 +903,6 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase):
         _("Metadata"), default=False, help_text=_("If true, will be excluded from search")
     )
 
-    files = JSONField(null=True, default=list, blank=True)
-
     blob = JSONField(null=True, default=dict, blank=True)
 
     subtype = models.CharField(max_length=128, null=True, blank=True)
@@ -2021,6 +2018,7 @@ class Link(models.Model):
     name = models.CharField(max_length=255, help_text=_('For example "View in Google Earth"'))
     mime = models.CharField(max_length=255, help_text=_('For example "text/xml"'))
     url = models.TextField(max_length=1000)
+    asset = models.ForeignKey("Asset", null=True, on_delete=models.SET_NULL)
 
     objects = LinkManager()
 
@@ -2127,3 +2125,43 @@ class GroupGeoLimit(models.Model):
 class ExtraMetadata(models.Model):
     resource = models.ForeignKey(ResourceBase, null=False, blank=False, on_delete=models.CASCADE)
     metadata = JSONField(null=True, default=dict, blank=True)
+
+
+class Asset(PolymorphicModel):
+    """
+    A generic data linked to a ResourceBase
+    """
+
+    title = models.CharField(max_length=255, null=False, blank=False)
+    description = models.TextField(null=True, blank=True)
+    type = models.CharField(max_length=255, null=False, blank=False)
+    owner = models.ForeignKey(get_user_model(), null=False, blank=False, on_delete=models.CASCADE)
+    created = models.DateTimeField(default=now)
+
+    class Meta:
+        verbose_name_plural = "Assets"
+
+    @classmethod
+    def storage_manager(cls, kind):
+        return StorageManagerFactory(kind)
+
+    def __str__(self) -> str:
+        return super().__str__()
+
+
+class LocalAsset(Asset):
+    """
+    Local resource, will replace the files
+    """
+
+    location = models.JSONField(default=list, blank=True)
+
+    class Meta:
+        verbose_name_plural = "Local assets"
+
+    @classmethod
+    def get_storage_manager(cls):
+        return super().storage_manager("local")
+
+    def __str__(self) -> str:
+        return super().__str__()

geonode/documents/tasks.py

@@ -104,8 +104,18 @@ def create_document_thumbnail(self, object_id):
     centering = (0.5, 0.5)
 
     doc_path = None
-    if document.files:
-        doc_path = storage_manager.path(document.files[0])
+
+    # get asset of the resource
+    original_link = document.link_set.filter(link_type="original").first()
+    if original_link is None:
+        raise Exception("Asset not assigned yet, cannot generate thumbnail")
+
+    asset = original_link.asset
+
+    storage_manager = asset.get_storage_manager()
+
+    if asset:
+        doc_path = storage_manager.path(asset.location[0])
     elif document.doc_url:
         doc_path = document.doc_url
         remove_tmp_file = True

geonode/documents/views.py

@@ -48,9 +48,10 @@
 from geonode.decorators import check_keyword_write_perms
 from geonode.security.utils import get_user_visible_groups, AdvancedSecurityWorkflowManager
 from geonode.base.forms import CategoryForm, TKeywordForm, ThesaurusAvailableForm
-from geonode.base.models import Thesaurus, TopicCategory
+from geonode.base.models import LocalAsset, Thesaurus, TopicCategory
 from geonode.base import enumerations
 
+from geonode.storage.assets import asset_manager
 from .utils import get_download_response
 
 from .models import Document
@@ -157,11 +158,28 @@ def form_valid(self, form):
         doc_form = form.cleaned_data
 
         file = doc_form.pop("doc_file", None)
+        # generate asset
         if file:
             tempdir = mkdtemp()
+
             dirname = os.path.basename(tempdir)
+
+            storage_manager = LocalAsset.get_storage_manager()
+
             filepath = storage_manager.save(f"{dirname}/{file.name}", file)
             storage_path = storage_manager.path(filepath)
+
+            # generation of the local asset
+            payload = {
+                "title": doc_form.get("title", file.name),
+                "owner": self.request.user,
+                "description": doc_form.pop("abstract", None),
+                "location": [storage_path],
+                "type": "document",
+            }
+
+            asset = asset_manager.create(LocalAsset, **payload)
+
             self.object = resource_manager.create(
                 None,
                 resource_type=Document,
@@ -170,11 +188,14 @@ def form_valid(self, form):
                     doc_url=doc_form.pop("doc_url", None),
                     title=doc_form.pop("title", file.name),
                     extension=doc_form.pop("extension", None),
-                    files=[storage_path],
+                    asset=asset,
                 ),
             )
             if tempdir != os.path.dirname(storage_path):
                 shutil.rmtree(tempdir, ignore_errors=True)
+
+            asset_manager.assign(asset=asset, resource=self.object)
+
         else:
             self.object = resource_manager.create(
                 None,

geonode/resource/manager.py

@@ -313,6 +313,7 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de
         if resource_type.objects.filter(uuid=uuid).exists():
             return resource_type.objects.filter(uuid=uuid).get()
         uuid = uuid or str(uuid4())
+        _asset = defaults.pop("asset", None)
         _resource, _created = resource_type.objects.get_or_create(uuid=uuid, defaults=defaults)
         if _resource and _created:
             _resource.set_processing_state(enumerations.STATE_RUNNING)
@@ -323,7 +324,7 @@ def create(self, uuid: str, /, resource_type: typing.Optional[object] = None, de
                         uuid, resource_type=resource_type, defaults=defaults
                     )
                 _resource.save()
-                resourcebase_post_save(_resource.get_real_instance())
+                resourcebase_post_save(_resource.get_real_instance(), asset=_asset)
                 _resource.set_processing_state(enumerations.STATE_PROCESSED)
             except Exception as e:
                 logger.exception(e)
@@ -503,11 +504,11 @@ def copy(
                             defaults.pop("name")
                     _resource.save()
                     for lr in LinkedResource.get_linked_resources(source=instance.pk, is_internal=False):
-                        LinkedResource.objects.get_or_create(
+                        LinkedResource.object.get_or_create(
                             source_id=_resource.pk, target_id=lr.target.pk, internal=False
                         )
                     for lr in LinkedResource.get_linked_resources(target=instance.pk, is_internal=False):
-                        LinkedResource.objects.get_or_create(
+                        LinkedResource.object.get_or_create(
                             source_id=lr.source.pk, target_id=_resource.pk, internal=False
                         )
 

geonode/resource/utils.py

@@ -322,9 +322,9 @@ def get_alternate_name(instance):
 
 def document_post_save(instance, *args, **kwargs):
     instance.csw_type = "document"
-
-    if instance.files:
-        _, extension = os.path.splitext(os.path.basename(instance.files[0]))
+    asset = kwargs.get("asset", None)
+    if asset:
+        _, extension = os.path.splitext(os.path.basename(asset.location[0]))
         instance.extension = extension[1:]
         doc_type_map = DOCUMENT_TYPE_MAP
         doc_type_map.update(getattr(settings, "DOCUMENT_TYPE_MAP", {}))
@@ -344,7 +344,7 @@ def document_post_save(instance, *args, **kwargs):
     mime = mime_type_map.get(ext, "text/plain")
     url = None
 
-    if instance.id and instance.files:
+    if instance.id and asset:
         name = "Hosted Document"
         site_url = settings.SITEURL.rstrip("/") if settings.SITEURL.startswith("http") else settings.SITEURL
         url = f"{site_url}{reverse('document_download', args=(instance.id,))}"