From eb7843a849baf3001d9f13478a1dd7bcbb12b8c0 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 8 Nov 2022 11:49:05 +0100 Subject: [PATCH 1/7] [Fixes #10270] Document creation via API v2 --- geonode/base/models.py | 6 ++- geonode/documents/api/exceptions.py | 26 +++++++++ geonode/documents/api/serializers.py | 26 +++++---- geonode/documents/api/tests.py | 81 ++++++++++++++++++++++++++++ geonode/documents/api/views.py | 56 +++++++++++++++++-- 5 files changed, 179 insertions(+), 16 deletions(-) create mode 100644 geonode/documents/api/exceptions.py diff --git a/geonode/base/models.py b/geonode/base/models.py index 99207f07269..31bb13c8439 100644 --- a/geonode/base/models.py +++ b/geonode/base/models.py @@ -777,7 +777,7 @@ class ResourceBase(PolymorphicModel, PermissionLevelMixin, ItemBase): extra_metadata_help_text = _( 'Additional metadata, must be in format [ {"metadata_key": "metadata_value"}, {"metadata_key": "metadata_value"} ]') # internal fields - uuid = models.CharField(max_length=36, unique=True, default=str(uuid.uuid4)) + uuid = models.CharField(max_length=36, unique=True, default=uuid.uuid4) title = models.CharField(_('title'), max_length=255, help_text=_( 'name by which the cited resource is known')) abstract = models.TextField( @@ -1233,7 +1233,9 @@ def save(self, notify=False, *args, **kwargs): self.pk = self.id = _next_value - if not self.uuid or len(self.uuid) == 0 or callable(self.uuid): + if isinstance(self.uuid, uuid.UUID): + self.uuid = str(self.uuid) + elif not self.uuid or callable(self.uuid) or len(self.uuid) == 0: self.uuid = str(uuid.uuid4()) super().save(*args, **kwargs) diff --git a/geonode/documents/api/exceptions.py b/geonode/documents/api/exceptions.py new file mode 100644 index 00000000000..4850255b1d9 --- /dev/null +++ b/geonode/documents/api/exceptions.py @@ -0,0 +1,26 @@ +######################################################################### +# +# Copyright (C) 2022 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +from rest_framework.exceptions import APIException + + +class DocumentException(APIException): + status_code = 400 + default_detail = "invalid document" + default_code = "document_exception" + category = "document_api" diff --git a/geonode/documents/api/serializers.py b/geonode/documents/api/serializers.py index 665b6bf514a..7c21cb35f10 100644 --- a/geonode/documents/api/serializers.py +++ b/geonode/documents/api/serializers.py @@ -16,26 +16,30 @@ # along with this program. If not, see . # ######################################################################### -from geonode.documents.models import Document -from geonode.base.api.serializers import ResourceBaseSerializer - import logging +from dynamic_rest.fields.fields import DynamicComputedField +from geonode.base.api.serializers import ResourceBaseSerializer +from geonode.documents.models import Document + logger = logging.getLogger(__name__) -class DocumentSerializer(ResourceBaseSerializer): +class GeonodeFilePathField(DynamicComputedField): + + def get_attribute(self, instance): + return instance.files + +class DocumentSerializer(ResourceBaseSerializer): def __init__(self, *args, **kwargs): # Instantiate the superclass normally super().__init__(*args, **kwargs) + file_path = GeonodeFilePathField() + class Meta: model = Document - name = 'document' - view_name = 'documents-list' - fields = ( - 'pk', 'uuid', 'name', 'href', - 'subtype', 'extension', 'mime_type', - 'executions' - ) + name = "document" + view_name = "documents-list" + fields = ("pk", "uuid", "name", "href", "subtype", "extension", "mime_type", "executions", "file_path") diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index 971148a30a6..4ed7bd60239 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -16,16 +16,22 @@ # along with this program. If not, see . # ######################################################################### +import os +from django.contrib.auth import get_user_model import logging from urllib.parse import urljoin +from django.test import override_settings from django.urls import reverse +from mock import patch from rest_framework.test import APITestCase from guardian.shortcuts import assign_perm, get_anonymous_user +from geonode import settings from geonode.documents.models import Document from geonode.base.populate_test_data import create_models +from geonode.documents.api.exceptions import DocumentException logger = logging.getLogger(__name__) @@ -42,6 +48,10 @@ def setUp(self): create_models(b'document') create_models(b'map') create_models(b'dataset') + self.admin = get_user_model().objects.get(username="admin") + self.url = reverse('documents-list') + self.invalid_file_path = f"{settings.PROJECT_ROOT}/tests/data/thesaurus.rdf" + self.valid_file_path = f"{settings.PROJECT_ROOT}/base/fixtures/test_xml.xml" def test_documents(self): """ @@ -72,3 +82,74 @@ def test_documents(self): # import json # logger.error(f"{json.dumps(layers_data)}") + + def test_creation_return_error_if_file_is_not_passed(self): + ''' + If file_path is not available, should raise error + ''' + self.client.force_login(self.admin) + payload = { + "document": { + "title": "New document", + "metadata_only": True + } + } + expected = {'success': False, 'errors': ['This field is required.'], 'code': 'invalid'} + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(400, actual.status_code) + self.assertDictEqual(expected, actual.json()) + + def test_creation_return_error_if_file_is_none(self): + ''' + If file_path is not available, should raise error + ''' + self.client.force_login(self.admin) + payload = { + "document": { + "title": "New document", + "metadata_only": True, + "file_path": None + } + } + expected = {'success': False, 'errors': ['This field may not be null.'], 'code': 'invalid'} + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(400, actual.status_code) + self.assertDictEqual(expected, actual.json()) + + def test_creation_should_rase_exec_for_unsupported_files(self): + self.client.force_login(self.admin) + payload = { + "document": { + "title": "New document", + "metadata_only": True, + "file_path": self.invalid_file_path + } + } + expected = {'success': False, 'errors': ['The file provided is not in the supported extension file list'], 'code': 'document_exception'} + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(400, actual.status_code) + self.assertDictEqual(expected, actual.json()) + + def test_creation_should_create_the_doc(self): + ''' + If file_path is not available, should raise error + ''' + self.client.force_login(self.admin) + payload = { + "document": { + "title": "New document for testing", + "metadata_only": True, + "file_path": self.valid_file_path + } + } + actual = self.client.post(self.url, data=payload, format="json") + self.assertEqual(201, actual.status_code) + cloned_path = actual.json().get("document", {}).get("file_path", "")[0] + extension = actual.json().get("document", {}).get("extension", "") + self.assertTrue(os.path.exists(cloned_path)) + self.assertEqual('xml', extension) + self.assertTrue(Document.objects.filter(title="New document for testing").exists()) + + if cloned_path: + os.remove(cloned_path) + \ No newline at end of file diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index 5121fab1912..cfc4490bd0b 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -16,8 +16,10 @@ # along with this program. If not, see . # ######################################################################### -from drf_spectacular.utils import extend_schema +from pkgutil import extend_path +from drf_spectacular.utils import extend_schema +from pathlib import Path from dynamic_rest.viewsets import DynamicModelViewSet from dynamic_rest.filters import DynamicFilterBackend, DynamicSortingFilter @@ -25,15 +27,20 @@ from rest_framework.permissions import IsAuthenticatedOrReadOnly from rest_framework.authentication import SessionAuthentication, BasicAuthentication from oauth2_provider.contrib.rest_framework import OAuth2Authentication +from geonode import settings from geonode.base.api.filters import DynamicSearchFilter, ExtentFilter from geonode.base.api.pagination import GeoNodeApiPagination from geonode.base.api.permissions import UserHasPerms +from geonode.documents.api.exceptions import DocumentException from geonode.documents.models import Document from geonode.base.models import ResourceBase from geonode.base.api.serializers import ResourceBaseSerializer +from geonode.storage.manager import StorageManager +from geonode.resource.manager import resource_manager +from geonode.documents.views import ALLOWED_DOC_TYPES from .serializers import DocumentSerializer from .permissions import DocumentPermissionsFilter @@ -46,9 +53,9 @@ class DocumentViewSet(DynamicModelViewSet): """ API endpoint that allows documents to be viewed or edited. """ - http_method_names = ['get', 'patch', 'put'] + http_method_names = ['get', 'patch', 'put', 'post'] authentication_classes = [SessionAuthentication, BasicAuthentication, OAuth2Authentication] - permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms] + permission_classes = [IsAuthenticatedOrReadOnly, UserHasPerms(perms_dict={"default": {"POST": ["base.add_resourcebase"]}})] filter_backends = [ DynamicFilterBackend, DynamicSortingFilter, DynamicSearchFilter, ExtentFilter, DocumentPermissionsFilter @@ -57,6 +64,49 @@ class DocumentViewSet(DynamicModelViewSet): serializer_class = DocumentSerializer pagination_class = GeoNodeApiPagination + def perform_create(self, serializer): + ''' + Function to create document via API v2. + The API expect this kind of JSON: + { + "document": { + "title": "New document", + "metadata_only": true, + "file_path": "/home/mattia/example.json" + } + } + File path rappresent the filepath where the file to upload is saved. + Is going to be cloned by the storage manager + ''' + manager = None + serializer.is_valid(raise_exception=True) + _has_file = serializer.validated_data.pop("file_path", None) + extension = serializer.validated_data.pop("extension", None) + + if not _has_file: + raise DocumentException(detail="A filepath must be speficied") + + if not extension: + extension = Path(_has_file).suffix.replace(".", "") + + if extension not in settings.ALLOWED_DOCUMENT_TYPES: + raise DocumentException("The file provided is not in the supported extension file list") + + try: + manager = StorageManager(remote_files={"base_file": _has_file}) + manager.clone_remote_files() + files = manager.get_retrieved_paths() + + resource = serializer.save(**{"owner": self.request.user, "extension": extension, "files": [files.get("base_file")]}) + + resource.handle_moderated_uploads() + resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=False) + return resource + except Exception as e: + if manager: + manager.delete_retrieved_paths() + raise e + @extend_schema(methods=['get'], responses={200: ResourceBaseSerializer(many=True)}, description="API endpoint allowing to retrieve the DocumentResourceLink(s).") @action(detail=True, methods=['get']) From 03f840bf85bda72b53313320aa676e25a94fd365 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 8 Nov 2022 11:50:16 +0100 Subject: [PATCH 2/7] [Fixes #10270] Document creation via API v2 --- geonode/documents/api/tests.py | 6 +----- geonode/documents/api/views.py | 6 ++---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index 4ed7bd60239..e4248632087 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -21,17 +21,14 @@ import logging from urllib.parse import urljoin -from django.test import override_settings from django.urls import reverse -from mock import patch from rest_framework.test import APITestCase from guardian.shortcuts import assign_perm, get_anonymous_user from geonode import settings from geonode.documents.models import Document from geonode.base.populate_test_data import create_models -from geonode.documents.api.exceptions import DocumentException logger = logging.getLogger(__name__) @@ -149,7 +146,6 @@ def test_creation_should_create_the_doc(self): self.assertTrue(os.path.exists(cloned_path)) self.assertEqual('xml', extension) self.assertTrue(Document.objects.filter(title="New document for testing").exists()) - + if cloned_path: os.remove(cloned_path) - \ No newline at end of file diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index cfc4490bd0b..1eb36893b53 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -17,7 +17,6 @@ # ######################################################################### -from pkgutil import extend_path from drf_spectacular.utils import extend_schema from pathlib import Path from dynamic_rest.viewsets import DynamicModelViewSet @@ -40,7 +39,6 @@ from geonode.storage.manager import StorageManager from geonode.resource.manager import resource_manager -from geonode.documents.views import ALLOWED_DOC_TYPES from .serializers import DocumentSerializer from .permissions import DocumentPermissionsFilter @@ -91,8 +89,8 @@ def perform_create(self, serializer): if extension not in settings.ALLOWED_DOCUMENT_TYPES: raise DocumentException("The file provided is not in the supported extension file list") - - try: + + try: manager = StorageManager(remote_files={"base_file": _has_file}) manager.clone_remote_files() files = manager.get_retrieved_paths() From a382721a2b7bcc0a055d70b0bc15ba9c5bb67f5f Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Tue, 8 Nov 2022 11:52:44 +0100 Subject: [PATCH 3/7] [Fixes #10270] Document creation via API v2 --- .../0085_alter_resourcebase_uuid.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 geonode/base/migrations/0085_alter_resourcebase_uuid.py diff --git a/geonode/base/migrations/0085_alter_resourcebase_uuid.py b/geonode/base/migrations/0085_alter_resourcebase_uuid.py new file mode 100644 index 00000000000..b7ef374172d --- /dev/null +++ b/geonode/base/migrations/0085_alter_resourcebase_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.16 on 2022-11-08 10:52 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0084_remove_comments_from_actions'), + ] + + operations = [ + migrations.AlterField( + model_name='resourcebase', + name='uuid', + field=models.CharField(default=uuid.uuid4, max_length=36, unique=True), + ), + ] From 252b16c2f7133ea621dd35c1c66077434508b70c Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 9 Nov 2022 15:07:43 +0100 Subject: [PATCH 4/7] [Fixes #10270] Document creation via API v2 --- geonode/documents/api/serializers.py | 11 +++++++++-- geonode/documents/api/views.py | 26 +++++++++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/geonode/documents/api/serializers.py b/geonode/documents/api/serializers.py index 7c21cb35f10..0ece3efbb8a 100644 --- a/geonode/documents/api/serializers.py +++ b/geonode/documents/api/serializers.py @@ -31,15 +31,22 @@ def get_attribute(self, instance): return instance.files +class DocumentFieldField(DynamicComputedField): + + def get_attribute(self, instance): + return instance.files + + class DocumentSerializer(ResourceBaseSerializer): def __init__(self, *args, **kwargs): # Instantiate the superclass normally super().__init__(*args, **kwargs) - file_path = GeonodeFilePathField() + file_path = GeonodeFilePathField(required=False) + doc_file = DocumentFieldField(required=False) class Meta: model = Document name = "document" view_name = "documents-list" - fields = ("pk", "uuid", "name", "href", "subtype", "extension", "mime_type", "executions", "file_path") + fields = ("pk", "uuid", "name", "href", "subtype", "extension", "mime_type", "executions", "file_path", "doc_file") diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index 1eb36893b53..0302f872a97 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -65,6 +65,9 @@ class DocumentViewSet(DynamicModelViewSet): def perform_create(self, serializer): ''' Function to create document via API v2. + file_path: path to the file + doc_file: the open file + The API expect this kind of JSON: { "document": { @@ -74,18 +77,24 @@ def perform_create(self, serializer): } } File path rappresent the filepath where the file to upload is saved. - Is going to be cloned by the storage manager + + or can be also a form-data: + curl --location --request POST 'http://localhost:8000/api/v2/documents' \ + --form 'title="Super Title2"' \ + --form 'doc_file=@"/C:/Users/user/Pictures/BcMc-a6T9IM.jpg"' \ + --form 'metadata_only="False"' ''' manager = None serializer.is_valid(raise_exception=True) - _has_file = serializer.validated_data.pop("file_path", None) + _has_file = serializer.validated_data.pop("file_path", None) or serializer.validated_data.pop("doc_file", None) extension = serializer.validated_data.pop("extension", None) if not _has_file: - raise DocumentException(detail="A filepath must be speficied") + raise DocumentException(detail="A file path or a file must be speficied") if not extension: - extension = Path(_has_file).suffix.replace(".", "") + filename = _has_file if isinstance(_has_file, str) else _has_file.name + extension = Path(filename).suffix.replace(".", "") if extension not in settings.ALLOWED_DOCUMENT_TYPES: raise DocumentException("The file provided is not in the supported extension file list") @@ -95,7 +104,14 @@ def perform_create(self, serializer): manager.clone_remote_files() files = manager.get_retrieved_paths() - resource = serializer.save(**{"owner": self.request.user, "extension": extension, "files": [files.get("base_file")]}) + resource = serializer.save( + **{ + "owner": self.request.user, + "extension": extension, + "files": [files.get("base_file")], + "resource_type": "document" + } + ) resource.handle_moderated_uploads() resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=False) From f39de60517ff2c152fd89f9290e68101c0062a09 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Wed, 9 Nov 2022 16:33:46 +0100 Subject: [PATCH 5/7] [Fixes #10270] Document creation via API v2 --- geonode/documents/api/tests.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/geonode/documents/api/tests.py b/geonode/documents/api/tests.py index e4248632087..aff90ad8661 100644 --- a/geonode/documents/api/tests.py +++ b/geonode/documents/api/tests.py @@ -91,7 +91,7 @@ def test_creation_return_error_if_file_is_not_passed(self): "metadata_only": True } } - expected = {'success': False, 'errors': ['This field is required.'], 'code': 'invalid'} + expected = {'success': False, 'errors': ['A file path or a file must be speficied'], 'code': 'document_exception'} actual = self.client.post(self.url, data=payload, format="json") self.assertEqual(400, actual.status_code) self.assertDictEqual(expected, actual.json()) @@ -105,10 +105,11 @@ def test_creation_return_error_if_file_is_none(self): "document": { "title": "New document", "metadata_only": True, - "file_path": None + "file_path": None, + "doc_file": None } } - expected = {'success': False, 'errors': ['This field may not be null.'], 'code': 'invalid'} + expected = {'success': False, 'errors': ['A file path or a file must be speficied'], 'code': 'document_exception'} actual = self.client.post(self.url, data=payload, format="json") self.assertEqual(400, actual.status_code) self.assertDictEqual(expected, actual.json()) From dd61c2319f07a9e92605d56549cdaaacf99a0dc0 Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 10 Nov 2022 11:21:17 +0100 Subject: [PATCH 6/7] [Fixes #10270] Document creation via API v2 --- geonode/documents/api/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index 0302f872a97..522f7d03ad6 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -36,6 +36,7 @@ from geonode.base.models import ResourceBase from geonode.base.api.serializers import ResourceBaseSerializer +from geonode.resource.utils import resourcebase_post_save from geonode.storage.manager import StorageManager from geonode.resource.manager import resource_manager @@ -112,7 +113,10 @@ def perform_create(self, serializer): "resource_type": "document" } ) - + + resource.set_missing_info() + resourcebase_post_save(resource.get_real_instance()) + resource_manager.set_permissions(None, instance=resource, permissions=None, created=True) resource.handle_moderated_uploads() resource_manager.set_thumbnail(resource.uuid, instance=resource, overwrite=False) return resource From 28ae5bcc6b686e61ffbc5568e92ce11a6ca0ec3f Mon Sep 17 00:00:00 2001 From: mattiagiupponi Date: Thu, 10 Nov 2022 11:58:28 +0100 Subject: [PATCH 7/7] [Fixes #10270] Document creation via API v2 --- geonode/documents/api/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geonode/documents/api/views.py b/geonode/documents/api/views.py index 522f7d03ad6..c8ee34ec65f 100644 --- a/geonode/documents/api/views.py +++ b/geonode/documents/api/views.py @@ -113,7 +113,7 @@ def perform_create(self, serializer): "resource_type": "document" } ) - + resource.set_missing_info() resourcebase_post_save(resource.get_real_instance()) resource_manager.set_permissions(None, instance=resource, permissions=None, created=True)