fix(ingest/bigquery): Not setting platform instance for bigquery plat…

…form resources (#11659)
datahub-project · Oct 17, 2024 · cf3634a · cf3634a
1 parent 68cd17b
commit cf3634a
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 50 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py
@@ -65,9 +65,6 @@
 # We can't use close as it is not called if the ingestion is not successful
 def cleanup(config: BigQueryV2Config) -> None:
     if config._credentials_path is not None:
-        logger.debug(
-            f"Deleting temporary credential file at {config._credentials_path}"
-        )
         os.unlink(config._credentials_path)
 
 

diff --git a/...a-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py b/...a-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_platform_resource_helper.py
@@ -42,7 +42,7 @@ def platform_resource_key(self) -> PlatformResourceKey:
         return PlatformResourceKey(
             platform="bigquery",
             resource_type="BigQueryLabelInfo",
-            platform_instance=self.project,
+            platform_instance=None,
             primary_key=self.label.primary_key(),
         )
 

diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json
@@ -201,7 +201,7 @@
 },
 {
     "entityType": "platformResource",
-    "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2",
+    "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b",
     "changeType": "UPSERT",
     "aspectName": "platformResourceInfo",
     "aspect": {
@@ -221,30 +221,29 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "bigquery-2022_02_03-07_00_00",
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "platformResource",
-    "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2",
+    "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:bigquery",
-            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,project-id-1)"
+            "platform": "urn:li:dataPlatform:bigquery"
         }
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "bigquery-2022_02_03-07_00_00",
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
-    "entityType": "dataset",
-    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)",
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:11f75a60f0dbd414676852e46a45e39b",
     "changeType": "UPSERT",
     "aspectName": "status",
     "aspect": {
@@ -254,13 +253,13 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "bigquery-2022_02_03-07_00_00",
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "platformResource",
-    "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3",
+    "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916",
     "changeType": "UPSERT",
     "aspectName": "platformResourceInfo",
     "aspect": {
@@ -280,19 +279,50 @@
     },
     "systemMetadata": {
         "lastObserved": 1643871600000,
-        "runId": "bigquery-2022_02_03-07_00_00",
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
         "lastRunId": "no-run-id-provided"
     }
 },
 {
     "entityType": "platformResource",
-    "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3",
+    "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916",
     "changeType": "UPSERT",
     "aspectName": "dataPlatformInstance",
     "aspect": {
         "json": {
-            "platform": "urn:li:dataPlatform:bigquery",
-            "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:bigquery,project-id-1)"
+            "platform": "urn:li:dataPlatform:bigquery"
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "platformResource",
+    "entityUrn": "urn:li:platformResource:99b34051bd90d28d922b0e107277a916",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
+        }
+    },
+    "systemMetadata": {
+        "lastObserved": 1643871600000,
+        "runId": "bigquery-2022_02_03-07_00_00-2j2qqv",
+        "lastRunId": "no-run-id-provided"
+    }
+},
+{
+    "entityType": "dataset",
+    "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)",
+    "changeType": "UPSERT",
+    "aspectName": "status",
+    "aspect": {
+        "json": {
+            "removed": false
         }
     },
     "systemMetadata": {
@@ -395,38 +425,6 @@
         "lastRunId": "no-run-id-provided"
     }
 },
-{
-    "entityType": "platformResource",
-    "entityUrn": "urn:li:platformResource:79d443a7956814fdab2168e11392bbf2",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "bigquery-2022_02_03-07_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
-{
-    "entityType": "platformResource",
-    "entityUrn": "urn:li:platformResource:0a8c87e84bd90486c4fd57bbae6557e3",
-    "changeType": "UPSERT",
-    "aspectName": "status",
-    "aspect": {
-        "json": {
-            "removed": false
-        }
-    },
-    "systemMetadata": {
-        "lastObserved": 1643871600000,
-        "runId": "bigquery-2022_02_03-07_00_00",
-        "lastRunId": "no-run-id-provided"
-    }
-},
 {
     "entityType": "dataset",
     "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:bigquery,project-id-1.bigquery-dataset-1.table-1,PROD)",