fix(ingestion): add datasource check for tableau

* Check before appending to datasource_for_csql
datahub-project · Apr 13, 2022 · c99dba0 · c99dba0
1 parent 60b505d
commit c99dba0
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 15 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau.py
@@ -418,21 +418,18 @@ def _create_lineage_from_csql_datasource(
         self, csql_urn: str, csql_datasource: List[dict]
     ) -> Iterable[MetadataWorkUnit]:
         for datasource in csql_datasource:
-            if datasource is not None:
-                datasource_urn = builder.make_dataset_urn(
-                    self.platform, datasource.get("id", ""), self.config.env
-                )
-                upstream_csql = UpstreamClass(
-                    dataset=csql_urn,
-                    type=DatasetLineageTypeClass.TRANSFORMED,
-                )
+            datasource_urn = builder.make_dataset_urn(
+                self.platform, datasource.get("id", ""), self.config.env
+            )
+            upstream_csql = UpstreamClass(
+                dataset=csql_urn,
+                type=DatasetLineageTypeClass.TRANSFORMED,
+            )
 
-                upstream_lineage = UpstreamLineage(upstreams=[upstream_csql])
-                yield self.get_metadata_change_proposal(
-                    datasource_urn,
-                    aspect_name="upstreamLineage",
-                    aspect=upstream_lineage,
-                )
+            upstream_lineage = UpstreamLineage(upstreams=[upstream_csql])
+            yield self.get_metadata_change_proposal(
+                datasource_urn, aspect_name="upstreamLineage", aspect=upstream_lineage
+            )
 
     def _create_lineage_to_upstream_tables(
         self, csql_urn: str, columns: List[dict]

diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py b/metadata-ingestion/src/datahub/ingestion/source/tableau_common.py
@@ -475,7 +475,7 @@ def get_unique_custom_sql(custom_sql_list: List[dict]) -> List[dict]:
         for column in custom_sql.get("columns", []):
             for field in column.get("referencedByFields", []):
                 datasource = field.get("datasource")
-                if datasource not in datasource_for_csql:
+                if datasource not in datasource_for_csql and datasource is not None:
                     datasource_for_csql.append(datasource)
 
         unique_csql["datasources"] = datasource_for_csql