fix(ingest/nifi): remove duplicate upstream jobs (datahub-project#10849)

aviv-julienjehannet · Jul 17, 2024 · 190ad3a · 190ad3a
1 parent 1f3551a
commit 190ad3a
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/nifi.py b/metadata-ingestion/src/datahub/ingestion/source/nifi.py
@@ -822,7 +822,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
             outgoing = list(
                 filter(lambda x: x[0] == component.id, self.nifi_flow.connections)
             )
-            inputJobs = []
+            inputJobs = set()
             jobProperties = None
 
             if component.nifi_type is NifiType.PROCESSOR:
@@ -873,7 +873,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                         dataset_urn,
                     )
                 else:
-                    inputJobs.append(
+                    inputJobs.add(
                         builder.make_data_job_urn_with_flow(flow_urn, incoming_from)
                     )
 
@@ -953,7 +953,7 @@ def construct_workunits(self) -> Iterable[MetadataWorkUnit]:  # noqa: C901
                 job_properties=jobProperties,
                 inlets=list(component.inlets.keys()),
                 outlets=list(component.outlets.keys()),
-                inputJobs=inputJobs,
+                inputJobs=list(inputJobs),
                 status=component.status,
             )