Skip to content

Commit

Permalink
[#2920] Fix check for natural keys duplicates in ZGW imports
Browse files Browse the repository at this point in the history
    - The check for natural keys duplicates produced false positives
      because the same `omschrijving` can be used for different types
      of ZGW objects (e.g. "Afgehandel" for status and resulaat).
      The check for duplicate natural keys is now scoped to the type
      of ZGW configuration.
  • Loading branch information
pi-sigma committed Dec 4, 2024
1 parent 3192304 commit e2f1391
Show file tree
Hide file tree
Showing 2 changed files with 15 additions and 5 deletions.
7 changes: 4 additions & 3 deletions src/open_inwoner/openzaak/import_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,7 +334,7 @@ def from_jsonl_stream_or_string(cls, stream_or_string: IO | str) -> Self:

rows_successfully_processed = 0
import_errors = []
natural_keys_seen = set()
natural_keys_seen = defaultdict(set)
for line in cls._lines_iter_from_jsonl_stream_or_string(stream_or_string):
try:
(deserialized_object,) = serializers.deserialize(
Expand All @@ -351,10 +351,11 @@ def from_jsonl_stream_or_string(cls, stream_or_string: IO | str) -> Self:
import_errors.append(error)
else:
source_config = deserialized_object.object
if (natural_key := source_config.natural_key()) in natural_keys_seen:
natural_key = source_config.natural_key()
if natural_key in natural_keys_seen[source_config.__class__.__name__]:
import_errors.append(ZGWImportError.from_jsonl(line))
continue
natural_keys_seen.add(natural_key)
natural_keys_seen[source_config.__class__.__name__].add(natural_key)
try:
match source_config:
case CatalogusConfig():
Expand Down
13 changes: 11 additions & 2 deletions src/open_inwoner/openzaak/tests/test_import_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,13 @@ def __init__(self, count=0):
zaaktype_uuids=[self.original_uuid],
description="",
)
self.ztc_resultaat_2 = ZaakTypeResultaatTypeConfigFactory(
zaaktype_config=self.ztc,
resultaattype_url=self.original_url,
omschrijving="status omschrijving", # test dupes across models
zaaktype_uuids=[self.original_uuid],
description="",
)
self.ztiotc = ZaakTypeInformatieObjectTypeConfigFactory(
zaaktype_config=self.ztc,
informatieobjecttype_url=self.original_url,
Expand Down Expand Up @@ -344,6 +351,7 @@ def setUp(self):
]
self.json_dupes = [
'{"model": "openzaak.zaaktypestatustypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "statustype_url": "https://bar.maykinmedia.nl", "omschrijving": "status omschrijving", "statustekst": "statustekst nieuw", "zaaktype_uuids": "[]", "status_indicator": "", "status_indicator_text": "", "document_upload_description": "", "description": "status", "notify_status_change": true, "action_required": false, "document_upload_enabled": true, "call_to_action_url": "", "call_to_action_text": "", "case_link_text": ""}}',
'{"model": "openzaak.zaaktyperesultaattypeconfig", "fields": {"zaaktype_config": ["ztc-id-a-0", "DM-0", "123456789"], "resultaattype_url": "https://bar.maykinmedia.nl", "omschrijving": "status omschrijving", "zaaktype_uuids": "[]", "description": "description new"}}',
]
self.jsonl = "\n".join(self.json_lines)
self.jsonl_with_dupes = "\n".join(self.json_lines + self.json_dupes)
Expand Down Expand Up @@ -574,12 +582,13 @@ def test_import_jsonl_update_reports_duplicate_natural_keys_in_upload_file(self)
)
import_expected = dataclasses.asdict(
ZGWConfigImport(
total_rows_processed=6,
total_rows_processed=7,
catalogus_configs_imported=1,
zaaktype_configs_imported=1,
zaak_informatie_object_type_configs_imported=1,
zaak_status_type_configs_imported=1,
zaak_resultaat_type_configs_imported=1,
# resultaat_type_config with "status omschrijving" should be imported
zaak_resultaat_type_configs_imported=2,
import_errors=[expected_error],
),
)
Expand Down

0 comments on commit e2f1391

Please sign in to comment.