diff --git a/src/open_inwoner/openzaak/admin.py b/src/open_inwoner/openzaak/admin.py index ffae93e571..9df74ee3e1 100644 --- a/src/open_inwoner/openzaak/admin.py +++ b/src/open_inwoner/openzaak/admin.py @@ -10,6 +10,7 @@ from django.template.defaultfilters import filesizeformat from django.template.response import TemplateResponse from django.urls import path, reverse +from django.utils.html import format_html, format_html_join from django.utils.translation import gettext_lazy as _, ngettext from import_export.admin import ImportExportMixin @@ -174,11 +175,30 @@ def process_file_view(self, request): self.message_user( request, _( - "Successfully processed %(num_rows)d items" - % {"num_rows": import_result.total_rows_processed} + "%(num_rows)d item(s) processed in total, with %(error_rows)d failing row(s)." + % { + "num_rows": import_result.total_rows_processed, + "error_rows": len(import_result.import_errors), + } ), - messages.SUCCESS, + messages.SUCCESS + if not import_result.import_errors + else messages.WARNING, ) + if errors := import_result.import_errors: + msgs_deduped = set(error.__str__() for error in errors) + error_msg_iterator = ([msg] for msg in msgs_deduped) + + error_msg_html = format_html_join( + "\n", "
  • {}
    • ", error_msg_iterator + ) + error_msg_html_ordered = format_html( + _("It was not possible to import the following items:") + + f"
      {error_msg_html}
    " + ) + self.message_user( + request, error_msg_html_ordered, messages.ERROR + ) return HttpResponseRedirect( reverse( diff --git a/src/open_inwoner/openzaak/import_export.py b/src/open_inwoner/openzaak/import_export.py index f2318ecf15..8eaea2414e 100644 --- a/src/open_inwoner/openzaak/import_export.py +++ b/src/open_inwoner/openzaak/import_export.py @@ -6,8 +6,11 @@ from typing import IO, Any, Generator, Self from urllib.parse import urlparse +from django.apps import apps from django.core import serializers +from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.files.storage import Storage +from django.core.serializers.base import DeserializationError from django.db import transaction from django.db.models import QuerySet @@ -22,6 +25,132 @@ logger = logging.getLogger(__name__) +class ZGWImportError(Exception): + @classmethod + def extract_error_data(cls, exc: Exception, jsonl: str): + exc_source = type(exc.__context__) + data = json.loads(jsonl) if jsonl else {} + source_config = apps.get_model(data["model"]) + + # error type + if exc_source is CatalogusConfig.DoesNotExist or source_config.DoesNotExist: + error_type = ObjectDoesNotExist + if exc_source is source_config.MultipleObjectsReturned: + error_type = MultipleObjectsReturned + + # metadata about source_config + items = [] + fields = data.get("fields", None) + if source_config is CatalogusConfig: + items = [ + f"Domein = {fields['domein']}", + f"Rsin = {fields['rsin']}", + ] + if source_config is ZaakTypeConfig: + items = [ + f"Identificatie = {fields['identificatie']}", + f"Catalogus domein = {fields['catalogus'][0]}", + f"Catalogus rsin = {fields['catalogus'][1]}", + ] + if source_config in { + ZaakTypeStatusTypeConfig, + ZaakTypeResultaatTypeConfig, + ZaakTypeInformatieObjectTypeConfig, + }: + items = [ + f"omschrijving = {fields['omschrijving']}", + f"ZaakTypeConfig identificatie = {fields['zaaktype_config'][0]}", + f"Catalogus domein = {fields['zaaktype_config'][1]}", + f"Catalogus rsin = {fields['zaaktype_config'][2]}", + ] + + return { + "error_type": error_type, + "source_config_name": source_config.__name__, + "info": ", ".join(items), + } + + @classmethod + def from_exception_and_jsonl(cls, exception: Exception, jsonl: str) -> Self: + error_data = cls.extract_error_data(exception, jsonl) + + error_template = ( + "%(source_config_name)s not found in target environment: %(info)s" + ) + if error_data["error_type"] is MultipleObjectsReturned: + error_template = "Got multiple results for %(source_config_name)s: %(info)s" + + return cls(error_template % error_data) + + +def check_catalogus_config_exists(source_config: CatalogusConfig, jsonl: str): + try: + CatalogusConfig.objects.get_by_natural_key( + domein=source_config.domein, rsin=source_config.rsin + ) + except CatalogusConfig.MultipleObjectsReturned as exc: + raise ZGWImportError.from_exception_and_jsonl(exc, jsonl) + except CatalogusConfig.DoesNotExist as exc: + raise ZGWImportError.from_exception_and_jsonl(exc, jsonl) + + +def _update_config(source, target, exclude_fields): + for field in source._meta.fields: + field_name = field.name + + if field_name in exclude_fields: + continue + + val = getattr(source, field_name, None) + setattr(target, field_name, val) + target.save() + + +def _update_zaaktype_config(source_config: ZaakTypeConfig, jsonl: str): + try: + target = ZaakTypeConfig.objects.get_by_natural_key( + identificatie=source_config.identificatie, + catalogus_domein=source_config.catalogus.domein, + catalogus_rsin=source_config.catalogus.rsin, + ) + except ZaakTypeConfig.MultipleObjectsReturned as exc: + raise ZGWImportError.from_exception_and_jsonl(exc, jsonl) + except (CatalogusConfig.DoesNotExist, ZaakTypeConfig.DoesNotExist) as exc: + raise ZGWImportError.from_exception_and_jsonl(exc, jsonl) + else: + exclude_fields = [ + "id", + "catalogus", + "urls", + "zaaktype_uuids", + ] + _update_config(source_config, target, exclude_fields) + + +def _update_nested_zgw_config( + source_config: ZaakTypeStatusTypeConfig + | ZaakTypeResultaatTypeConfig + | ZaakTypeInformatieObjectTypeConfig, + exclude_fields: list[str], + jsonl: str, +): + zaaktype_config_identificatie = source_config.zaaktype_config.identificatie + catalogus_domein = source_config.zaaktype_config.catalogus.domein + catalogus_rsin = source_config.zaaktype_config.catalogus.rsin + + try: + target = source_config.__class__.objects.get_by_natural_key( + omschrijving=source_config.omschrijving, + zaaktype_config_identificatie=zaaktype_config_identificatie, + catalogus_domein=catalogus_domein, + catalogus_rsin=catalogus_rsin, + ) + except (source_config.DoesNotExist, source_config.MultipleObjectsReturned) as exc: + raise ZGWImportError.from_exception_and_jsonl(exc, jsonl) + else: + _update_config(source_config, target, exclude_fields) + + @dataclasses.dataclass(frozen=True) class CatalogusConfigExport: """Gather and export CatalogusConfig(s) and all associated relations.""" @@ -113,9 +242,10 @@ class CatalogusConfigImport: total_rows_processed: int = 0 catalogus_configs_imported: int = 0 zaaktype_configs_imported: int = 0 - zaak_inormatie_object_type_configs_imported: int = 0 + zaak_informatie_object_type_configs_imported: int = 0 zaak_status_type_configs_imported: int = 0 zaak_resultaat_type_configs_imported: int = 0 + import_errors: list | None = None @staticmethod def _get_url_root(url: str) -> str: @@ -149,90 +279,85 @@ def _lines_iter_from_jsonl_stream_or_string( # Reset the stream in case it gets re-used lines.seek(0) - @classmethod - def _rewrite_jsonl_url_references( - cls, stream_or_string: IO | str - ) -> Generator[str, Any, None]: - # The assumption is that the exporting and importing instance both have - # a `Service` with the same slug as the `Service` referenced in the - # `configued_from` attribute of the imported CatalogusConfig. The - # assumption is further that all URLs in the imported objects are - # prefixed by an URL that matches the API root in the service. Because - # of this, the import file will contain URLs with a base URL pointing to - # the `api_root`` of the `configured_from` Service on the _source_ - # instance, and has to be re-written to match the `api_root` of the - # `configured_from` Service on the _target_ instance. Put differently, - # we assume that we are migrating ZGW objects that _do not differ_ as - # far as the ZGW objects themselves are concerned (apart from the URL, - # they essentially point to the same ZGW backend), but that they _do_ - # differ in terms of additional model fields that do not have their - # source of truth in the ZGW backends. - # - # This expectation is also encoded in our API clients: you can only - # fetch ZGW objects using the ApePie clients if the root of those - # objects matches the configured API root. - - base_url_mapping = {} - for deserialized_object in serializers.deserialize( - "jsonl", - filter( - lambda row: ('"model": "openzaak.catalogusconfig"' in row), - cls._lines_iter_from_jsonl_stream_or_string(stream_or_string), - ), - use_natural_foreign_keys=True, - use_natural_primary_keys=True, - ): - object_type: str = deserialized_object.object.__class__.__name__ - - if object_type == "CatalogusConfig": - target_base_url = cls._get_url_root( - deserialized_object.object.service.api_root - ) - source_base_url = cls._get_url_root(deserialized_object.object.url) - base_url_mapping[source_base_url] = target_base_url - else: - # https://www.xkcd.com/2200/ - logger.error( - "Tried to filter for catalogus config objects, but also got: %s", - object_type, - ) - - for line in cls._lines_iter_from_jsonl_stream_or_string(stream_or_string): - source_url_found = False - for source, target in base_url_mapping.items(): - line = line.replace(source, target) - source_url_found = True - - if not source_url_found: - raise ValueError("Unable to rewrite ZGW urls") - - yield line - @classmethod @transaction.atomic() def from_jsonl_stream_or_string(cls, stream_or_string: IO | str) -> Self: model_to_counter_mapping = { "CatalogusConfig": "catalogus_configs_imported", "ZaakTypeConfig": "zaaktype_configs_imported", - "ZaakTypeInformatieObjectTypeConfig": "zaak_inormatie_object_type_configs_imported", + "ZaakTypeInformatieObjectTypeConfig": "zaak_informatie_object_type_configs_imported", "ZaakTypeStatusTypeConfig": "zaak_status_type_configs_imported", "ZaakTypeResultaatTypeConfig": "zaak_resultaat_type_configs_imported", } - object_type_counts = defaultdict(int) - for deserialized_object in serializers.deserialize( - "jsonl", - cls._rewrite_jsonl_url_references(stream_or_string), - use_natural_foreign_keys=True, - use_natural_primary_keys=True, - ): - deserialized_object.save() - object_type = deserialized_object.object.__class__.__name__ - object_type_counts[object_type] += 1 + rows_successfully_processed = 0 + import_errors = [] + for line in cls._lines_iter_from_jsonl_stream_or_string(stream_or_string): + try: + (deserialized_object,) = serializers.deserialize( + "jsonl", + line, + use_natural_primary_keys=True, + use_natural_foreign_keys=True, + ) + except DeserializationError as exc: + error = ZGWImportError.from_exception_and_jsonl(exc, line) + logger.error(error) + import_errors.append(error) + else: + source_config = deserialized_object.object + try: + match source_config: + case CatalogusConfig(): + check_catalogus_config_exists( + source_config=source_config, jsonl=line + ) + case ZaakTypeConfig(): + _update_zaaktype_config( + source_config=source_config, jsonl=line + ) + case ZaakTypeInformatieObjectTypeConfig(): + exclude_fields = [ + "id", + "zaaktype_config", + "zaaktype_uuids", + "informatieobjecttype_url", + ] + _update_nested_zgw_config( + source_config, exclude_fields, line + ) + case ZaakTypeStatusTypeConfig(): + exclude_fields = [ + "id", + "zaaktype_config", + "zaaktype_uuids", + "statustype_url", + ] + _update_nested_zgw_config( + source_config, exclude_fields, line + ) + case ZaakTypeResultaatTypeConfig(): + exclude_fields = [ + "id", + "zaaktype_config", + "zaaktype_uuids", + "resultaattype_url", + ] + _update_nested_zgw_config( + source_config, exclude_fields, line + ) + except ZGWImportError as exc: + logger.error(exc) + import_errors.append(exc) + else: + object_type = source_config.__class__.__name__ + object_type_counts[object_type] += 1 + rows_successfully_processed += 1 creation_kwargs = { - "total_rows_processed": sum(object_type_counts.values()), + "total_rows_processed": rows_successfully_processed + len(import_errors), + "import_errors": import_errors, } for model_name, counter_field in model_to_counter_mapping.items(): diff --git a/src/open_inwoner/openzaak/managers.py b/src/open_inwoner/openzaak/managers.py index 6f515dde64..7e578f427f 100644 --- a/src/open_inwoner/openzaak/managers.py +++ b/src/open_inwoner/openzaak/managers.py @@ -93,7 +93,7 @@ class ZaakTypeInformatieObjectTypeConfigQueryset(models.QuerySet): def get_by_natural_key( self, omschrijving, - zaak_type_config_identificatie, + zaaktype_config_identificatie, catalogus_domein, catalogus_rsin, ): @@ -101,7 +101,7 @@ def get_by_natural_key( return self.get( zaaktype_config=ZaakTypeConfig.objects.get( - identificatie=zaak_type_config_identificatie, + identificatie=zaaktype_config_identificatie, catalogus__domein=catalogus_domein, catalogus__rsin=catalogus_rsin, ), @@ -248,7 +248,7 @@ class ZaakTypeResultaatTypeConfigManger(models.Manager): def get_by_natural_key( self, omschrijving, - zaak_type_config_identificatie, + zaaktype_config_identificatie, catalogus_domein, catalogus_rsin, ): @@ -256,7 +256,7 @@ def get_by_natural_key( return self.get( zaaktype_config=ZaakTypeConfig.objects.get( - identificatie=zaak_type_config_identificatie, + identificatie=zaaktype_config_identificatie, catalogus__domein=catalogus_domein, catalogus__rsin=catalogus_rsin, ), diff --git a/src/open_inwoner/openzaak/models.py b/src/open_inwoner/openzaak/models.py index d0b34fff27..85eaf4f157 100644 --- a/src/open_inwoner/openzaak/models.py +++ b/src/open_inwoner/openzaak/models.py @@ -4,7 +4,6 @@ from typing import Protocol, cast from urllib.parse import urlparse -from django.core.exceptions import ValidationError from django.db import models, transaction from django.db.models import UniqueConstraint from django.utils import timezone diff --git a/src/open_inwoner/openzaak/tests/test_admin.py b/src/open_inwoner/openzaak/tests/test_admin.py index 5ec4fd7b8f..fe904f1630 100644 --- a/src/open_inwoner/openzaak/tests/test_admin.py +++ b/src/open_inwoner/openzaak/tests/test_admin.py @@ -175,28 +175,34 @@ def test_export_action_returns_correct_export(self): msg="Response should be valid JSONL matching the input object", ) - @mock.patch( - "open_inwoner.openzaak.import_export.CatalogusConfigImport.import_from_jsonl_file_in_django_storage" - ) - def test_import_flow_reports_success(self, m) -> None: + def test_import_flow_reports_success(self) -> None: + service = ServiceFactory.create(slug="service-0") + CatalogusConfigFactory.create( + url="https://foo.0.maykinmedia.nl", + domein="DM-0", + rsin="123456789", + service=service, + ) + import_lines = [ + b'{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "DM-0", "rsin": "123456789", "service": ["service-0"]}}', + ] + import_line = b"\n".join(import_lines) + mock_file = Upload("dump.json", import_line, "application/json") + form = self.app.get( reverse( "admin:upload_zgw_import_file", ), user=self.user, ).form - form["zgw_export_file"] = self.mock_file + form["zgw_export_file"] = mock_file response = form.submit().follow() - self.assertEqual(m.call_count, 1) - self.assertEqual(m.call_args[0][0], "zgw_import_dump_2024-08-14-17-50-01.json") - self.assertTrue(isinstance(m.call_args[0][1], PrivateMediaFileSystemStorage)) - messages = [str(msg) for msg in response.context["messages"]] self.assertEqual( messages, - [_("Successfully processed 1 items")], + [_("1 item(s) processed in total, with 0 failing row(s).")], ) self.assertFalse( PrivateMediaFileSystemStorage().exists( @@ -251,3 +257,97 @@ def test_import_flow_errors_reports_failure_to_user(self, m) -> None: "admin:upload_zgw_import_file", ), ) + + def test_import_flow_reports_errors(self) -> None: + import_lines = [ + b'{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "DM-0", "rsin": "123456789", "service": ["service-0"]}}', + b'{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.0.maykinmedia.nl\\"]", "catalogus": ["DM-0", "123456789"], "identificatie": "ztc-id-a-0", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": true, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', + b'{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "informatieobjecttype_url": "http://foo.0.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": true, "document_notification_enabled": true}}', + b'{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "statustekst nieuw", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', + b'{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": "description new"}}', + b'{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "bogus", "statustekst": "bogus", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', + ] + import_line = b"\n".join(import_lines) + mock_file = Upload("dump.json", import_line, "application/json") + + form = self.app.get( + reverse( + "admin:upload_zgw_import_file", + ), + user=self.user, + ).form + form["zgw_export_file"] = mock_file + + response = form.submit().follow() + + messages = [str(msg) for msg in response.context["messages"]] + self.assertEqual(len(messages), 2) + self.assertEqual( + _("6 item(s) processed in total, with 6 failing row(s)."), + messages[0], + ) + self.assertIn( + "ZaakTypeConfig not found in target environment: Identificatie = ztc-id-a-0, " + "Catalogus domein = DM-0, Catalogus rsin = 123456789", + messages[1], + ) + self.assertIn( + "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = status omschrijving, " + "ZaakTypeConfig identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789", + messages[1], + ) + self.assertIn( + "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = bogus, ZaakTypeConfig " + "identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789", + messages[1], + ) + self.assertIn( + "ZaakTypeInformatieObjectTypeConfig not found in target environment: omschrijving = informatieobject, " + "ZaakTypeConfig identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789", + messages[1], + ) + self.assertIn( + "CatalogusConfig not found in target environment: Domein = DM-0, Rsin = 123456789", + messages[1], + ) + self.assertIn( + "ZaakTypeResultaatTypeConfig not found in target environment: omschrijving = resultaat, " + "ZaakTypeConfig identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789", + messages[1], + ) + + def test_import_flow_reports_partial_errors(self) -> None: + service = ServiceFactory.create(slug="service-0") + CatalogusConfigFactory.create( + url="https://foo.0.maykinmedia.nl", + domein="DM-0", + rsin="123456789", + service=service, + ) + import_lines = [ + b'{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "DM-0", "rsin": "123456789", "service": ["service-0"]}}', + b'{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.0.maykinmedia.nl\\"]", "catalogus": ["DM-0", "123456789"], "identificatie": "ztc-id-a-0", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": true, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', + ] + import_line = b"\n".join(import_lines) + mock_file = Upload("dump.json", import_line, "application/json") + + form = self.app.get( + reverse( + "admin:upload_zgw_import_file", + ), + user=self.user, + ).form + form["zgw_export_file"] = mock_file + + response = form.submit().follow() + + messages = [str(msg) for msg in response.context["messages"]] + self.assertEqual(len(messages), 2) + self.assertEqual( + _("2 item(s) processed in total, with 1 failing row(s)."), + messages[0], + ) + self.assertEqual( + "It was not possible to import the following items:
    1. ZaakTypeConfig not found in target environment: Identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789
    ", + messages[1], + ) diff --git a/src/open_inwoner/openzaak/tests/test_import_export.py b/src/open_inwoner/openzaak/tests/test_import_export.py index 55bb2959d2..c950b825b3 100644 --- a/src/open_inwoner/openzaak/tests/test_import_export.py +++ b/src/open_inwoner/openzaak/tests/test_import_export.py @@ -1,4 +1,6 @@ +import dataclasses import io +import uuid from django.core.files.storage.memory import InMemoryStorage from django.test import TestCase @@ -15,6 +17,7 @@ ZaakTypeStatusTypeConfig, ) +from ..import_export import ZGWImportError from .factories import ( CatalogusConfigFactory, ServiceFactory, @@ -27,9 +30,11 @@ class ZGWExportImportMockData: def __init__(self, count=0): + self.original_url = f"https://foo.{count}.maykinmedia.nl" + self.original_uuid = "a1591906-3368-470a-a957-4b8634c275a1" self.service = ServiceFactory(slug=f"service-{count}") self.catalogus = CatalogusConfigFactory( - url=f"https://foo.{count}.maykinmedia.nl", + url=self.original_url, domein=f"DM-{count}", rsin="123456789", service=self.service, @@ -44,14 +49,14 @@ def __init__(self, count=0): contact_form_enabled=False, contact_subject_code="", relevante_zaakperiode=None, - urls=[f"https://foo.{count}.maykinmedia.nl"], + urls=[self.original_url], ) self.ztc_status = ZaakTypeStatusTypeConfigFactory( zaaktype_config=self.ztc, - statustype_url=f"https://foo.{count}.maykinmedia.nl", + statustype_url=self.original_url, omschrijving="status omschrijving", statustekst="", - zaaktype_uuids=[], + zaaktype_uuids=[self.original_uuid], status_indicator="", status_indicator_text="", document_upload_description="", @@ -65,15 +70,16 @@ def __init__(self, count=0): ) self.ztc_resultaat = ZaakTypeResultaatTypeConfigFactory( zaaktype_config=self.ztc, - resultaattype_url=f"https://foo.{count}.maykinmedia.nl", + resultaattype_url=self.original_url, omschrijving="resultaat", - zaaktype_uuids=[], + zaaktype_uuids=[self.original_uuid], description="", ) self.ztiotc = ZaakTypeInformatieObjectTypeConfigFactory( zaaktype_config=self.ztc, - informatieobjecttype_url=f"http://foo.{count}.maykinmedia.nl", + informatieobjecttype_url=self.original_url, omschrijving="informatieobject", + zaaktype_uuids=[self.original_uuid], ) @@ -185,9 +191,9 @@ def test_export_catalogus_configs(self): "model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": { "zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], - "informatieobjecttype_url": "http://foo.0.maykinmedia.nl", + "informatieobjecttype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "informatieobject", - "zaaktype_uuids": "[]", + "zaaktype_uuids": '["a1591906-3368-470a-a957-4b8634c275a1"]', "document_upload_enabled": False, "document_notification_enabled": False, }, @@ -199,7 +205,7 @@ def test_export_catalogus_configs(self): "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", - "zaaktype_uuids": "[]", + "zaaktype_uuids": '["a1591906-3368-470a-a957-4b8634c275a1"]', "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", @@ -218,7 +224,7 @@ def test_export_catalogus_configs(self): "zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "resultaat", - "zaaktype_uuids": "[]", + "zaaktype_uuids": '["a1591906-3368-470a-a957-4b8634c275a1"]', "description": "", }, }, @@ -241,17 +247,17 @@ def test_export_catalogus_configs_as_jsonl(self): "\n", '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.1.maykinmedia.nl\\"]", "catalogus": ["DM-1", "123456789"], "identificatie": "ztc-id-a-1", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', "\n", - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "informatieobjecttype_url": "http://foo.0.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', + '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "informatieobjecttype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[\\"a1591906-3368-470a-a957-4b8634c275a1\\"]", "document_upload_enabled": false, "document_notification_enabled": false}}', "\n", - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "informatieobjecttype_url": "http://foo.1.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', + '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "informatieobjecttype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[\\"a1591906-3368-470a-a957-4b8634c275a1\\"]", "document_upload_enabled": false, "document_notification_enabled": false}}', "\n", - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', + '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[\\"a1591906-3368-470a-a957-4b8634c275a1\\"]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', "\n", - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "statustype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', + '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "statustype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[\\"a1591906-3368-470a-a957-4b8634c275a1\\"]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', "\n", - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": ""}}', + '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[\\"a1591906-3368-470a-a957-4b8634c275a1\\"]", "description": ""}}', "\n", - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "resultaattype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": ""}}', + '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "resultaattype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[\\"a1591906-3368-470a-a957-4b8634c275a1\\"]", "description": ""}}', "\n", ] @@ -261,243 +267,302 @@ def test_export_catalogus_configs_as_jsonl(self): class TestCatalogusImport(TestCase): def setUp(self): self.storage = InMemoryStorage() - self.service = ServiceFactory( - slug="service-0", api_root="https://foo.0.maykinmedia.nl" - ) - self.other_service = ServiceFactory( - slug="service-1", api_root="https://foo.1.maykinmedia.nl" - ) - self.json_lines = [ - '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "DM-0", "rsin": "123456789", "service": ["service-0"]}}', - '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.1.maykinmedia.nl", "domein": "DM-1", "rsin": "123456789", "service": ["service-1"]}}', - '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.0.maykinmedia.nl\\"]", "catalogus": ["DM-0", "123456789"], "identificatie": "ztc-id-a-0", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.1.maykinmedia.nl\\"]", "catalogus": ["DM-1", "123456789"], "identificatie": "ztc-id-a-1", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "informatieobjecttype_url": "http://foo.0.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "informatieobjecttype_url": "http://foo.1.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "statustype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-1", "DM-1", "123456789"], "resultaattype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": ""}}', + '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://bar.maykinmedia.nl", "domein": "DM-0", "rsin": "123456789", "service": ["service-0"]}}', + '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://bar.maykinmedia.nl\\"]", "catalogus": ["DM-0", "123456789"], "identificatie": "ztc-id-a-0", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": true, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', + '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "informatieobjecttype_url": "https://bar.maykinmedia.nl", "omschrijving": "informatieobject", "zaaktype_uuids": "[]", "document_upload_enabled": true, "document_notification_enabled": true}}', + '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://bar.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "statustekst nieuw", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', + '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://bar.maykinmedia.nl", "omschrijving": "resultaat", "zaaktype_uuids": "[]", "description": "description new"}}', ] self.jsonl = "\n".join(self.json_lines) - def test_import_jsonl_creates_objects(self): + def test_import_jsonl_update_success(self): + mocks = ZGWExportImportMockData() self.storage.save("import.jsonl", io.StringIO(self.jsonl)) import_result = CatalogusConfigImport.import_from_jsonl_file_in_django_storage( "import.jsonl", self.storage ) + + # check import self.assertEqual( import_result, CatalogusConfigImport( - total_rows_processed=10, - catalogus_configs_imported=2, - zaaktype_configs_imported=2, - zaak_inormatie_object_type_configs_imported=2, - zaak_status_type_configs_imported=2, - zaak_resultaat_type_configs_imported=2, + total_rows_processed=5, + catalogus_configs_imported=1, + zaaktype_configs_imported=1, + zaak_informatie_object_type_configs_imported=1, + zaak_status_type_configs_imported=1, + zaak_resultaat_type_configs_imported=1, + import_errors=[], ), ) - self.assertEqual(CatalogusConfig.objects.count(), 2) - self.assertEqual(ZaakTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 2) + # check number of configs + self.assertEqual(CatalogusConfig.objects.count(), 1) + self.assertEqual(ZaakTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 1) - def test_import_jsonl_merges_objects(self): - CatalogusConfigFactory( - url="https://foo.0.maykinmedia.nl", - domein="FOO", - rsin="123456789", - service=self.service, + catalogus_config = CatalogusConfig.objects.get() + zt_config = ZaakTypeConfig.objects.get() + zt_informatie_objecttype_config = ( + ZaakTypeInformatieObjectTypeConfig.objects.get() ) - merge_line = '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "BAR", "rsin": "987654321", "service": ["service-0"]}}' - - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(merge_line) - - self.assertEqual(import_result.catalogus_configs_imported, 1) - self.assertEqual(import_result.total_rows_processed, 1) + zt_statustype_config = ZaakTypeStatusTypeConfig.objects.get() + zt_resultaattype_config = ZaakTypeResultaatTypeConfig.objects.get() + # check that urls are not overridden + self.assertEqual(catalogus_config.url, mocks.original_url) + self.assertEqual(zt_config.urls, [mocks.original_url]) self.assertEqual( - list(CatalogusConfig.objects.values_list("url", "domein", "rsin")), - [("https://foo.0.maykinmedia.nl", "BAR", "987654321")], - msg="Value of sole CatalogusConfig matches imported values, not original values", + zt_informatie_objecttype_config.informatieobjecttype_url, + mocks.original_url, ) + self.assertEqual(zt_statustype_config.statustype_url, mocks.original_url) + self.assertEqual(zt_resultaattype_config.resultaattype_url, mocks.original_url) - def test_bad_import_types(self): - for bad_type in (set(), list(), b""): - with self.assertRaises(ValueError): - CatalogusConfigImport.from_jsonl_stream_or_string(bad_type) - - def test_valid_input_types_are_accepted(self): - for input in ( - io.StringIO(self.jsonl), - io.BytesIO(self.jsonl.encode("utf-8")), - self.jsonl, - ): - with self.subTest(f"Input type {type(input)}"): - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(input) - self.assertEqual( - import_result, - CatalogusConfigImport( - total_rows_processed=10, - catalogus_configs_imported=2, - zaaktype_configs_imported=2, - zaak_inormatie_object_type_configs_imported=2, - zaak_status_type_configs_imported=2, - zaak_resultaat_type_configs_imported=2, - ), - ) - - self.assertEqual(CatalogusConfig.objects.count(), 2) - self.assertEqual(ZaakTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 2) - self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 2) + # check that zaaktype uuids are not overridden + self.assertEqual( + zt_informatie_objecttype_config.zaaktype_uuids, + [uuid.UUID(mocks.original_uuid)], + ) + self.assertEqual( + zt_statustype_config.zaaktype_uuids, [uuid.UUID(mocks.original_uuid)] + ) + self.assertEqual( + zt_resultaattype_config.zaaktype_uuids, [uuid.UUID(mocks.original_uuid)] + ) - def test_import_is_atomic(self): - bad_line = '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {}}\n' - bad_jsonl = self.jsonl + "\n" + bad_line + # check updated content + zaaktype_statustype_config = ZaakTypeStatusTypeConfig.objects.get() + self.assertEqual(zaaktype_statustype_config.statustekst, "statustekst nieuw") + self.assertEqual(zaaktype_statustype_config.notify_status_change, True) - try: - CatalogusConfigImport.from_jsonl_stream_or_string( - stream_or_string=bad_jsonl - ) - except Exception: - pass + zaaktype_resultaattype_config = ZaakTypeResultaatTypeConfig.objects.get() + self.assertEqual(zaaktype_resultaattype_config.description, "description new") - counts = ( - CatalogusConfig.objects.count(), - ZaakTypeConfig.objects.count(), - ZaakTypeInformatieObjectTypeConfig.objects.count(), - ZaakTypeStatusTypeConfig.objects.count(), - ZaakTypeResultaatTypeConfig.objects.count(), + zaaktype_informatie_objecttype_config = ( + ZaakTypeInformatieObjectTypeConfig.objects.get() ) - expected_counts = (0, 0, 0, 0, 0) - self.assertEqual( - counts, - expected_counts, - msg="Import should have merged, and not created new values", + zaaktype_informatie_objecttype_config.document_upload_enabled, True ) - - -class RewriteUrlsImportTests(TestCase): - def setUp(self): - self.service = ServiceFactory( - slug="constant-api-slug", api_root="http://one.maykinmedia.nl" + self.assertEqual( + zaaktype_informatie_objecttype_config.document_notification_enabled, True ) - import_lines = [ - '{"model": "openzaak.catalogusconfig", "fields": {"url": "http://one.maykinmedia.nl/catalogus/1", "domein": "ALLE", "rsin": "1234568", "service": ["constant-api-slug"]}}', - '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"http://one.maykinmedia.nl/types/1\\", \\"http://one.maykinmedia.nl/types/2\\"]", "catalogus": ["http://one.maykinmedia.nl/catalogus/1"], "identificatie": "zt-1", "omschrijving": "iGsHCEkCpEJyDLeAaytskGiAXSAPVVthCvOdbNdpZZcCciXFnZGltXFYsYigSkIZiaqMEvSPftMgIYyW", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://one.maykinmedia.nl/catalogus/1"], "informatieobjecttype_url": "http://one.maykinmedia.nl/iotype/1", "omschrijving": "IzNqfWpVpbyMEjSXTqQUlslqAUYFdILFlSDAelAkfTROWptqgIRCmaIoWCBMBAozsJLWxGoJqmBLPCHy", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://one.maykinmedia.nl/catalogus/1"], "statustype_url": "http://one.maykinmedia.nl/status-type/1", "omschrijving": "BHEJLQkSTdMPGtSzgnIbIdhMvFiNOBHmFQkRvLxHUkmafelprqCpcuAZzqMWBLgqNkGmXpzWPjhWqKjk", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://one.maykinmedia.nl/catalogus/1"], "resultaattype_url": "http://one.maykinmedia.nl/resultaat-type/1", "omschrijving": "", "zaaktype_uuids": "[]", "description": ""}}', + def test_import_jsonl_missing_statustype_config(self): + ZGWExportImportMockData() + + # missing ZaakTypeStatusTypeConfig + json_line_extra = [ + '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://foo.0.maykinmedia.nl", "omschrijving": "bogus", "statustekst": "bogus", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', ] - self.jsonl = "\n".join(import_lines) + json_lines = "\n".join(self.json_lines + json_line_extra) - def _create_fixtures(self, base_url: str): - catalogus = CatalogusConfigFactory( - url=f"{base_url}/catalogus/1", - service=self.service, - domein="ALLE", - rsin="1234568", - ) - zt = ZaakTypeConfigFactory( - catalogus=catalogus, - identificatie="zt-1", - urls=[ - f"{base_url}/types/1", - f"{base_url}/types/2", - ], - ) - ZaakTypeInformatieObjectTypeConfigFactory( - zaaktype_config=zt, - informatieobjecttype_url=f"{base_url}/iotype/1", + self.storage.save("import.jsonl", io.StringIO(json_lines)) + + # we use `asdict` and replace the Exceptions with string representations + # because for Exceptions raised from within dataclasses, equality ==/is identity + import_result = dataclasses.asdict( + CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + "import.jsonl", self.storage + ) ) - ZaakTypeStatusTypeConfigFactory( - zaaktype_config=zt, statustype_url=f"{base_url}/status-type/1" + expected_error = ZGWImportError( + "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = bogus, " + "ZaakTypeConfig identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789" ) - ZaakTypeResultaatTypeConfigFactory( - zaaktype_config=zt, - resultaattype_url=f"{base_url}/resultaat-type/1", + import_expected = dataclasses.asdict( + CatalogusConfigImport( + total_rows_processed=6, + catalogus_configs_imported=1, + zaaktype_configs_imported=1, + zaak_informatie_object_type_configs_imported=1, + zaak_status_type_configs_imported=1, + zaak_resultaat_type_configs_imported=1, + import_errors=[expected_error], + ), ) + import_result["import_errors"][0] = str(import_result["import_errors"][0]) + import_expected["import_errors"][0] = str(import_expected["import_errors"][0]) - def test_jsonl_url_rewrite(self): - self.service.api_root = "http://two.maykinmedia.nl" - self.service.save() + # check import + self.assertEqual(import_result, import_expected) - rewritten_lines = list( - CatalogusConfigImport._rewrite_jsonl_url_references(self.jsonl) - ) - expected_lines = [ - '{"model": "openzaak.catalogusconfig", "fields": {"url": "http://two.maykinmedia.nl/catalogus/1", "domein": "ALLE", "rsin": "1234568", "service": ["constant-api-slug"]}}', - '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"http://two.maykinmedia.nl/types/1\\", \\"http://two.maykinmedia.nl/types/2\\"]", "catalogus": ["http://two.maykinmedia.nl/catalogus/1"], "identificatie": "zt-1", "omschrijving": "iGsHCEkCpEJyDLeAaytskGiAXSAPVVthCvOdbNdpZZcCciXFnZGltXFYsYigSkIZiaqMEvSPftMgIYyW", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', - '{"model": "openzaak.zaaktypeinformatieobjecttypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://two.maykinmedia.nl/catalogus/1"], "informatieobjecttype_url": "http://two.maykinmedia.nl/iotype/1", "omschrijving": "IzNqfWpVpbyMEjSXTqQUlslqAUYFdILFlSDAelAkfTROWptqgIRCmaIoWCBMBAozsJLWxGoJqmBLPCHy", "zaaktype_uuids": "[]", "document_upload_enabled": false, "document_notification_enabled": false}}', - '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://two.maykinmedia.nl/catalogus/1"], "statustype_url": "http://two.maykinmedia.nl/status-type/1", "omschrijving": "BHEJLQkSTdMPGtSzgnIbIdhMvFiNOBHmFQkRvLxHUkmafelprqCpcuAZzqMWBLgqNkGmXpzWPjhWqKjk", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', - '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["zt-1", "http://two.maykinmedia.nl/catalogus/1"], "resultaattype_url": "http://two.maykinmedia.nl/resultaat-type/1", "omschrijving": "", "zaaktype_uuids": "[]", "description": ""}}', - ] + # check number of configs + self.assertEqual(CatalogusConfig.objects.count(), 1) + self.assertEqual(ZaakTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 1) - self.assertEqual( - rewritten_lines, - expected_lines, - msg="All URLs should be rewritten to match the target service root", - ) + def test_import_jsonl_update_statustype_config_missing_zt_config(self): + ZGWExportImportMockData() - def test_rewrite_target_diverges_from_existing_objects(self): - self._create_fixtures("http://one.maykinmedia.nl/") - self.service.api_root = "http://two.maykinmedia.nl" - self.service.save() + # import fails due to missing ZaakTypeConfig + json_line_extra = [ + '{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["bogus", "DM-1", "666666666"], "statustype_url": "https://foo.1.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}', + ] + json_lines = "\n".join(self.json_lines + json_line_extra) - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(self.jsonl) + self.storage.save("import.jsonl", io.StringIO(json_lines)) - self.assertEqual( - import_result, + # we use `asdict` and replace the Exceptions with string representations + # because for Exceptions raised from within dataclasses, equality ==/is identity + import_result = dataclasses.asdict( + CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + "import.jsonl", self.storage + ) + ) + expected_error = ZGWImportError( + "ZaakTypeStatusTypeConfig not found in target environment: omschrijving = status omschrijving, " + "ZaakTypeConfig identificatie = bogus, Catalogus domein = DM-1, Catalogus rsin = 666666666" + ) + import_expected = dataclasses.asdict( CatalogusConfigImport( - total_rows_processed=5, + total_rows_processed=6, catalogus_configs_imported=1, zaaktype_configs_imported=1, - zaak_inormatie_object_type_configs_imported=1, + zaak_informatie_object_type_configs_imported=1, zaak_status_type_configs_imported=1, zaak_resultaat_type_configs_imported=1, + import_errors=[expected_error], ), ) + import_result["import_errors"][0] = str(import_result["import_errors"][0]) + import_expected["import_errors"][0] = str(import_expected["import_errors"][0]) - counts = ( - CatalogusConfig.objects.count(), - ZaakTypeConfig.objects.count(), - ZaakTypeInformatieObjectTypeConfig.objects.count(), - ZaakTypeStatusTypeConfig.objects.count(), - ZaakTypeResultaatTypeConfig.objects.count(), - ) - expected_counts = (2, 2, 2, 2, 2) + # check import + self.assertEqual(import_result, import_expected) - self.assertEqual( - counts, - expected_counts, - msg="Import should have merged, and not created new values", - ) + # check number of configs + self.assertEqual(CatalogusConfig.objects.count(), 1) + self.assertEqual(ZaakTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeInformatieObjectTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeStatusTypeConfig.objects.count(), 1) + self.assertEqual(ZaakTypeResultaatTypeConfig.objects.count(), 1) - def test_rewrite_target_matches_from_existing_objects(self): - self.service.api_root = "http://two.maykinmedia.nl" - self.service.save() - self._create_fixtures("http://two.maykinmedia.nl") + def test_import_jsonl_update_reports_duplicates(self): + mocks = ZGWExportImportMockData() - import_result = CatalogusConfigImport.from_jsonl_stream_or_string(self.jsonl) - self.assertEqual( - import_result, + ZaakTypeResultaatTypeConfigFactory( + resultaattype_url=mocks.ztc_resultaat.resultaattype_url, + zaaktype_config=mocks.ztc_resultaat.zaaktype_config, + omschrijving=mocks.ztc_resultaat.omschrijving, + description=mocks.ztc_resultaat.description, + ) + self.storage.save("import.jsonl", io.StringIO(self.jsonl)) + + # we use `asdict` and replace the Exceptions with string representations + # because for Exceptions raised from within dataclasses, equality ==/is identity + import_result = dataclasses.asdict( + CatalogusConfigImport.import_from_jsonl_file_in_django_storage( + "import.jsonl", self.storage + ) + ) + expected_error = ZGWImportError( + "Got multiple results for ZaakTypeResultaatTypeConfig: omschrijving = resultaat, " + "ZaakTypeConfig identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789" + ) + import_expected = dataclasses.asdict( CatalogusConfigImport( total_rows_processed=5, catalogus_configs_imported=1, zaaktype_configs_imported=1, - zaak_inormatie_object_type_configs_imported=1, + zaak_informatie_object_type_configs_imported=1, zaak_status_type_configs_imported=1, - zaak_resultaat_type_configs_imported=1, + zaak_resultaat_type_configs_imported=0, + import_errors=[expected_error], ), ) + import_result["import_errors"][0] = str(import_result["import_errors"][0]) + import_expected["import_errors"][0] = str(import_expected["import_errors"][0]) + + # check import + self.assertEqual(import_result, import_expected) + + def test_import_jsonl_fails_with_catalogus_domein_rsin_mismatch(self): + service = ServiceFactory(slug="service-0") + CatalogusConfigFactory( + url="https://foo.0.maykinmedia.nl", + domein="FOO", + rsin="123456789", + service=service, + ) + import_lines = [ + '{"model": "openzaak.catalogusconfig", "fields": {"url": "https://foo.0.maykinmedia.nl", "domein": "BAR", "rsin": "987654321", "service": ["service-0"]}}', + '{"model": "openzaak.zaaktypeconfig", "fields": {"urls": "[\\"https://foo.0.maykinmedia.nl\\"]", "catalogus": ["DM-0", "123456789"], "identificatie": "ztc-id-a-0", "omschrijving": "zaaktypeconfig", "notify_status_changes": false, "description": "", "external_document_upload_url": "", "document_upload_enabled": false, "contact_form_enabled": false, "contact_subject_code": "", "relevante_zaakperiode": null}}', + ] + import_line = "\n".join(import_lines) + + with self.assertLogs( + logger="open_inwoner.openzaak.import_export", level="ERROR" + ) as cm: + import_result = CatalogusConfigImport.from_jsonl_stream_or_string( + import_line + ) + self.assertEqual( + cm.output, + [ + # error from trying to load existing CatalogusConfig + "ERROR:open_inwoner.openzaak.import_export:" + "CatalogusConfig not found in target environment: Domein = BAR, Rsin = 987654321", + # error from deserializing nested ZGW objects + "ERROR:open_inwoner.openzaak.import_export:" + "ZaakTypeConfig not found in target environment: Identificatie = ztc-id-a-0, Catalogus domein = DM-0, Catalogus rsin = 123456789", + ], + ) + + self.assertEqual(CatalogusConfig.objects.count(), 1) + + self.assertEqual(import_result.catalogus_configs_imported, 0) + self.assertEqual(import_result.total_rows_processed, 2) + + self.assertEqual( + list(CatalogusConfig.objects.values_list("url", "domein", "rsin")), + [("https://foo.0.maykinmedia.nl", "FOO", "123456789")], + msg="Value of sole CatalogusConfig matches imported values, not original values", + ) + + def test_bad_import_types(self): + for bad_type in (set(), list(), b""): + with self.assertRaises(ValueError): + CatalogusConfigImport.from_jsonl_stream_or_string(bad_type) + + def test_valid_input_types_are_accepted(self): + ZGWExportImportMockData() + + for input in ( + io.StringIO(self.jsonl), + io.BytesIO(self.jsonl.encode("utf-8")), + self.jsonl, + ): + with self.subTest(f"Input type {type(input)}"): + import_result = CatalogusConfigImport.from_jsonl_stream_or_string(input) + self.assertEqual( + import_result, + CatalogusConfigImport( + total_rows_processed=5, + catalogus_configs_imported=1, + zaaktype_configs_imported=1, + zaak_informatie_object_type_configs_imported=1, + zaak_status_type_configs_imported=1, + zaak_resultaat_type_configs_imported=1, + import_errors=[], + ), + ) + + def test_import_is_atomic(self): + bad_line = '{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {}}\n' + bad_jsonl = self.jsonl + "\n" + bad_line + + with self.assertRaises(KeyError): + CatalogusConfigImport.from_jsonl_stream_or_string( + stream_or_string=bad_jsonl + ) counts = ( CatalogusConfig.objects.count(), @@ -506,7 +571,7 @@ def test_rewrite_target_matches_from_existing_objects(self): ZaakTypeStatusTypeConfig.objects.count(), ZaakTypeResultaatTypeConfig.objects.count(), ) - expected_counts = (1, 1, 1, 1, 1) + expected_counts = (0, 0, 0, 0, 0) self.assertEqual( counts, diff --git a/src/open_inwoner/openzaak/tests/test_models.py b/src/open_inwoner/openzaak/tests/test_models.py index 3c12d340e7..402c05c78e 100644 --- a/src/open_inwoner/openzaak/tests/test_models.py +++ b/src/open_inwoner/openzaak/tests/test_models.py @@ -1,7 +1,6 @@ from datetime import timedelta from uuid import uuid4 -from django.db import IntegrityError from django.test import TestCase from freezegun import freeze_time @@ -30,12 +29,6 @@ class CatalogusConfigManagerTestCase(TestCase): - def test_fields_used_for_natural_key_are_unique(self): - - with self.assertRaises(IntegrityError): - for _ in range(2): - CatalogusConfigFactory(domein="test", rsin="1234") - def test_get_by_natural_key_returns_expected_instance(self): config = CatalogusConfigFactory() @@ -46,23 +39,10 @@ def test_get_by_natural_key_returns_expected_instance(self): def test_get_by_natural_key_not_found(self): with self.assertRaises(CatalogusConfig.DoesNotExist): - CatalogusConfig.objects.get_by_natural_key( - domein="test", rsin="1234" - ) + CatalogusConfig.objects.get_by_natural_key(domein="test", rsin="1234") class ZaakTypeConfigModelTestCase(TestCase): - def test_fields_used_for_natural_key_are_unique(self): - catalogus = CatalogusConfigFactory() - - with self.assertRaises(IntegrityError): - for _ in range(2): - ZaakTypeConfigFactory( - identificatie="AAAA", - catalogus__domein=catalogus.domein, - catalogus__rsin=catalogus.rsin, - ) - def test_get_by_natural_key_returns_expected_instance(self): zaak_type_config = ZaakTypeConfigFactory( identificatie="AAAA", @@ -107,15 +87,6 @@ def test_queryset_filter_case_type_with_catalog(self): class ZaakTypeStatusTypeConfigModelTestCase(TestCase): - def test_fields_used_for_natural_key_are_unique(self): - zt = ZaakTypeConfigFactory() - with self.assertRaises(IntegrityError): - for _ in range(2): - ZaakTypeStatusTypeConfigFactory( - zaaktype_config=zt, - omschrijving="test", - ) - def test_get_by_natural_key_returns_expected_instance(self): zt = ZaakTypeConfigFactory() zt_status_type_config = ZaakTypeStatusTypeConfigFactory( @@ -145,24 +116,17 @@ def test_get_by_natural_key_not_found(self): class ZaakTypeResultaatTypeConfigModelTestCase(TestCase): - def test_fields_used_for_natural_key_are_unique(self): - zt = ZaakTypeConfigFactory() - with self.assertRaises(IntegrityError): - for _ in range(2): - ZaakTypeResultaatTypeConfigFactory( - zaaktype_config=zt, omschrijving="test" - ) - def test_get_by_natural_key_returns_expected_instance(self): zt = ZaakTypeConfigFactory() zt_status_type_config = ZaakTypeResultaatTypeConfigFactory( - zaaktype_config=zt, omschrijving="test", + zaaktype_config=zt, + omschrijving="test", ) self.assertEqual( ZaakTypeResultaatTypeConfig.objects.get_by_natural_key( omschrijving="test", - zaak_type_config_identificatie=zt.identificatie, + zaaktype_config_identificatie=zt.identificatie, catalogus_domein=zt.catalogus.domein, catalogus_rsin=zt.catalogus.rsin, ), @@ -174,22 +138,13 @@ def test_get_by_natural_key_not_found(self): with self.assertRaises(ZaakTypeResultaatTypeConfig.DoesNotExist): ZaakTypeResultaatTypeConfig.objects.get_by_natural_key( omschrijving="bogus", - zaak_type_config_identificatie=zt.identificatie, + zaaktype_config_identificatie=zt.identificatie, catalogus_domein=zt.catalogus.domein, catalogus_rsin=zt.catalogus.rsin, ) class ZaakTypeInformatieObjectTypeConfigFactoryModelTestCase(TestCase): - def test_fields_used_for_natural_key_are_unique(self): - zt = ZaakTypeConfigFactory() - with self.assertRaises(IntegrityError): - for _ in range(2): - ZaakTypeInformatieObjectTypeConfigFactory( - zaaktype_config=zt, - omschrijving="test", - ) - def test_get_by_natural_key_returns_expected_instance(self): zt = ZaakTypeConfigFactory() zt_io_type = ZaakTypeInformatieObjectTypeConfigFactory( @@ -199,7 +154,7 @@ def test_get_by_natural_key_returns_expected_instance(self): self.assertEqual( ZaakTypeInformatieObjectTypeConfig.objects.get_by_natural_key( omschrijving="test", - zaak_type_config_identificatie=zt.identificatie, + zaaktype_config_identificatie=zt.identificatie, catalogus_domein=zt.catalogus.domein, catalogus_rsin=zt.catalogus.rsin, ), @@ -211,7 +166,7 @@ def test_get_by_natural_key_not_found(self): with self.assertRaises(ZaakTypeInformatieObjectTypeConfig.DoesNotExist): ZaakTypeInformatieObjectTypeConfig.objects.get_by_natural_key( omschrijving="bogus", - zaak_type_config_identificatie=zt.identificatie, + zaaktype_config_identificatie=zt.identificatie, catalogus_domein=zt.catalogus.domein, catalogus_rsin=zt.catalogus.rsin, ) diff --git a/src/open_inwoner/openzaak/tests/test_zgw_imports_command.py b/src/open_inwoner/openzaak/tests/test_zgw_imports_command.py index 3f091fad70..19db15d804 100644 --- a/src/open_inwoner/openzaak/tests/test_zgw_imports_command.py +++ b/src/open_inwoner/openzaak/tests/test_zgw_imports_command.py @@ -43,11 +43,8 @@ def setUpTestData(cls): def test_zgw_import_data_command(self, m): m.reset_mock() for root in self.roots: - CatalogMockData(root).install_mocks(m) InformationObjectTypeMockData(root).install_mocks(m) - # ZaakTypeMockData(root).install_mocks(m) - - # TODO: ADD CATALOGI like in iotypes + CatalogMockData(root).install_mocks(m) # run it to import our data out = StringIO()