From 37023bddab91fcb05eadc282b2d30f61e49d3853 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Wed, 29 May 2024 15:14:46 -0700 Subject: [PATCH] fix(ingest/dbt): only generate one subtype --- .../ingestion/source/dbt/dbt_common.py | 7 ------ .../dbt_enabled_with_schemas_mces_golden.json | 14 +++-------- .../dbt_test_column_meta_mapping_golden.json | 17 ++++--------- .../dbt/dbt_test_events_golden.json | 17 ++++--------- ...bt_test_test_model_performance_golden.json | 25 +++++++------------ ...th_complex_owner_patterns_mces_golden.json | 14 +++-------- ...th_data_platform_instance_mces_golden.json | 14 +++-------- ...h_non_incremental_lineage_mces_golden.json | 14 +++-------- ..._target_platform_instance_mces_golden.json | 14 +++-------- 9 files changed, 39 insertions(+), 97 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index ad68050444f194..b758f218e25866 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -45,7 +45,6 @@ from datahub.ingestion.api.source import MetadataWorkUnitProcessor from datahub.ingestion.api.source_helpers import auto_workunit from datahub.ingestion.api.workunit import MetadataWorkUnit -from datahub.ingestion.source.common.subtypes import DatasetSubTypes from datahub.ingestion.source.dbt.dbt_tests import ( DBTTest, DBTTestResult, @@ -1739,12 +1738,6 @@ def _create_subType_wu( return None subtypes: List[str] = [node.node_type.capitalize()] - if node.materialization == "table": - subtypes.append(DatasetSubTypes.TABLE) - - if node.node_type == "model" or node.node_type == "snapshot": - # We need to add the view subtype so that the view properties tab shows up in the UI. - subtypes.append(DatasetSubTypes.VIEW) return MetadataChangeProposalWrapper( entityUrn=node_datahub_urn, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json index ba381d688fbad3..54f4309d962aae 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_enabled_with_schemas_mces_golden.json @@ -7,8 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -372,9 +371,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -606,8 +603,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -943,9 +939,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json index a87b4134dc8989..04107bcde903ec 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_column_meta_mapping_golden.json @@ -7,8 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -337,9 +336,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -559,8 +556,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -888,9 +884,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -1107,8 +1101,7 @@ "aspect": { "json": { "typeNames": [ - "Snapshot", - "View" + "Snapshot" ] } }, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json index 7f8b018afb586d..4e5199aeec3893 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_events_golden.json @@ -7,9 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -233,9 +231,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -478,8 +474,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -633,8 +628,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -800,8 +794,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json index 66670e95402d91..60596547c3d508 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_test_model_performance_golden.json @@ -7,8 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -337,9 +336,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -565,7 +562,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an-aliased-view-for-monthly-billing,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1643871600000, + "time": 1663355198240, "actor": "urn:li:corpuser:datahub" } } @@ -685,8 +682,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -1020,7 +1016,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.an_aliased_view_for_payments,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1643871600000, + "time": 1663355198240, "actor": "urn:li:corpuser:datahub" } } @@ -1144,9 +1140,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -1349,7 +1343,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.payments_by_customer_by_month,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1643871600000, + "time": 1663355198240, "actor": "urn:li:corpuser:datahub" } } @@ -1468,8 +1462,7 @@ "aspect": { "json": { "typeNames": [ - "Snapshot", - "View" + "Snapshot" ] } }, @@ -1873,7 +1866,7 @@ "name": "just-some-random-id_urn:li:dataset:(urn:li:dataPlatform:dbt,pagila.public.customer_snapshot,PROD)", "type": "BATCH_SCHEDULED", "created": { - "time": 1643871600000, + "time": 1663355198240, "actor": "urn:li:corpuser:datahub" } } diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json index fcea12ecf8d3d5..4ec48e71badcdf 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_complex_owner_patterns_mces_golden.json @@ -7,8 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -336,9 +335,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -552,8 +549,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -889,9 +885,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json index c9fd7f5ba0d45e..9002001fde29ee 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_data_platform_instance_mces_golden.json @@ -7,8 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -338,9 +337,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -555,8 +552,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -893,9 +889,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json index f9c8a452c3cebb..d16542adaa0307 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_non_incremental_lineage_mces_golden.json @@ -7,8 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -337,9 +336,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -553,8 +550,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -890,9 +886,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, diff --git a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json index e0eb8a40a4d202..921af5cb3d1db3 100644 --- a/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json +++ b/metadata-ingestion/tests/integration/dbt/dbt_test_with_target_platform_instance_mces_golden.json @@ -7,8 +7,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -337,9 +336,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } }, @@ -553,8 +550,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "View" + "Model" ] } }, @@ -890,9 +886,7 @@ "aspect": { "json": { "typeNames": [ - "Model", - "Table", - "View" + "Model" ] } },