Skip to content

Commit

Permalink
fix(ingest): only populate audit stamps where accurate (#10604)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hsheth2
hsheth2 authored Jul 11, 2024
1 parent 44930df commit 82bd3c2
Show file tree
Hide file tree
Showing 9 changed files with 54 additions and 70 deletions.
2 changes: 1 addition & 1 deletion metadata-ingestion/src/datahub/ingestion/source/metabase.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ def construct_card_from_api_data(self, card_data: dict) -> Optional[ChartSnapsho
f"{last_edit_by.get('timestamp')}"
)
last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor),
created=None,
lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
)

Expand Down
4 changes: 2 additions & 2 deletions metadata-ingestion/src/datahub/ingestion/source/redash.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -555,7 +555,7 @@ def _get_dashboard_snapshot(self, dashboard_data, redash_version):
title = dashboard_data.get("name", "")

last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor),
created=None,
lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
)

Expand Down Expand Up @@ -694,7 +694,7 @@ def _get_chart_snapshot(self, query_data: Dict, viz_data: Dict) -> ChartSnapshot
title = f"{query_data.get('name')} {viz_data.get('name', '')}"

last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor),
created=None,
lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
)

Expand Down
6 changes: 1 addition & 5 deletions metadata-ingestion/src/datahub/ingestion/source/redshift/redshift.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -800,11 +800,7 @@ def gen_dataset_workunits(
lastModified=(
TimeStamp(time=int(table.last_altered.timestamp() * 1000))
if table.last_altered
else (
TimeStamp(time=int(table.created.timestamp() * 1000))
if table.created
else None
)
else None
),
description=table.comment,
qualifiedName=str(datahub_dataset_name),
Expand Down
6 changes: 1 addition & 5 deletions metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_schema_gen.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -762,11 +762,7 @@ def get_dataset_properties(
lastModified=(
TimeStamp(time=int(table.last_altered.timestamp() * 1000))
if table.last_altered is not None
else (
TimeStamp(time=int(table.created.timestamp() * 1000))
if table.created is not None
else None
)
else None
),
description=table.comment,
qualifiedName=f"{db_name}.{schema_name}.{table.name}",
Expand Down
12 changes: 4 additions & 8 deletions metadata-ingestion/src/datahub/ingestion/source/superset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,11 +273,9 @@ def construct_dashboard_from_api_data(self, dashboard_data):
dp.parse(dashboard_data.get("changed_on_utc", "now")).timestamp() * 1000
)
title = dashboard_data.get("dashboard_title", "")
# note: the API does not currently supply created_by usernames due to a bug, but we are required to
# provide a created AuditStamp to comply with ChangeAuditStamp model. For now, I sub in the last
# modified actor urn
# note: the API does not currently supply created_by usernames due to a bug
last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor),
created=None,
lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
)
dashboard_url = f"{self.config.display_uri}{dashboard_data.get('url', '')}"
Expand Down Expand Up @@ -380,11 +378,9 @@ def construct_chart_from_chart_data(self, chart_data):
)
title = chart_data.get("slice_name", "")

# note: the API does not currently supply created_by usernames due to a bug, but we are required to
# provide a created AuditStamp to comply with ChangeAuditStamp model. For now, I sub in the last
# modified actor urn
# note: the API does not currently supply created_by usernames due to a bug
last_modified = ChangeAuditStamps(
created=AuditStamp(time=modified_ts, actor=modified_actor),
created=None,
lastModified=AuditStamp(time=modified_ts, actor=modified_actor),
)
chart_type = chart_type_from_viz_type.get(chart_data.get("viz_type", ""))
Expand Down
12 changes: 6 additions & 6 deletions metadata-ingestion/tests/integration/metabase/metabase_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1639417592792,
"actor": "urn:li:corpuser:[email protected]"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1639417592792,
Expand Down Expand Up @@ -82,8 +82,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1636614000000,
"actor": "urn:li:corpuser:[email protected]"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1636614000000,
Expand Down Expand Up @@ -140,8 +140,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1685628119636,
"actor": "urn:li:corpuser:[email protected]"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1685628119636,
Expand Down
42 changes: 24 additions & 18 deletions metadata-ingestion/tests/integration/superset/golden_test_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
"datasets": [],
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_1"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand All @@ -44,7 +44,8 @@
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "superset-test"
"runId": "superset-test",
"lastRunId": "no-run-id-provided"
}
},
{
Expand Down Expand Up @@ -74,8 +75,8 @@
"datasets": [],
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_2"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand All @@ -90,7 +91,8 @@
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "superset-test"
"runId": "superset-test",
"lastRunId": "no-run-id-provided"
}
},
{
Expand All @@ -114,8 +116,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_1"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand All @@ -136,7 +138,8 @@
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "superset-test"
"runId": "superset-test",
"lastRunId": "no-run-id-provided"
}
},
{
Expand All @@ -160,8 +163,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_1"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand All @@ -182,7 +185,8 @@
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "superset-test"
"runId": "superset-test",
"lastRunId": "no-run-id-provided"
}
},
{
Expand All @@ -206,8 +210,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_2"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand All @@ -228,7 +232,8 @@
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "superset-test"
"runId": "superset-test",
"lastRunId": "no-run-id-provided"
}
},
{
Expand All @@ -252,8 +257,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_2"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand All @@ -274,7 +279,8 @@
},
"systemMetadata": {
"lastObserved": 1586847600000,
"runId": "superset-test"
"runId": "superset-test",
"lastRunId": "no-run-id-provided"
}
}
]
20 changes: 10 additions & 10 deletions metadata-ingestion/tests/integration/superset/golden_test_stateful_ingest.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@
"datasets": [],
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_1"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand Down Expand Up @@ -70,8 +70,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_1"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand Down Expand Up @@ -118,8 +118,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_1"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand Down Expand Up @@ -166,8 +166,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_2"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand Down Expand Up @@ -214,8 +214,8 @@
"description": "",
"lastModified": {
"created": {
"time": 1586847600000,
"actor": "urn:li:corpuser:test_username_2"
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
"lastModified": {
"time": 1586847600000,
Expand Down
20 changes: 5 additions & 15 deletions metadata-ingestion/tests/unit/test_redash_source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -489,9 +489,7 @@ def test_get_dashboard_snapshot_before_v10():
],
datasets=[],
lastModified=ChangeAuditStamps(
created=AuditStamp(
time=1628882055288, actor="urn:li:corpuser:unknown"
),
created=None,
lastModified=AuditStamp(
time=1628882055288, actor="urn:li:corpuser:unknown"
),
Expand Down Expand Up @@ -521,9 +519,7 @@ def test_get_dashboard_snapshot_after_v10():
],
datasets=[],
lastModified=ChangeAuditStamps(
created=AuditStamp(
time=1628882055288, actor="urn:li:corpuser:unknown"
),
created=None,
lastModified=AuditStamp(
time=1628882055288, actor="urn:li:corpuser:unknown"
),
Expand Down Expand Up @@ -551,9 +547,7 @@ def test_get_known_viz_chart_snapshot(mocked_data_source):
title="My Query Chart",
description="",
lastModified=ChangeAuditStamps(
created=AuditStamp(
time=1628882022544, actor="urn:li:corpuser:unknown"
),
created=None,
lastModified=AuditStamp(
time=1628882022544, actor="urn:li:corpuser:unknown"
),
Expand Down Expand Up @@ -584,9 +578,7 @@ def test_get_unknown_viz_chart_snapshot(mocked_data_source):
title="My Query Sankey",
description="",
lastModified=ChangeAuditStamps(
created=AuditStamp(
time=1628882009571, actor="urn:li:corpuser:unknown"
),
created=None,
lastModified=AuditStamp(
time=1628882009571, actor="urn:li:corpuser:unknown"
),
Expand Down Expand Up @@ -711,9 +703,7 @@ def test_get_chart_snapshot_parse_table_names_from_sql(mocked_data_source):
title="My Query Chart",
description="",
lastModified=ChangeAuditStamps(
created=AuditStamp(
time=1628882022544, actor="urn:li:corpuser:unknown"
),
created=None,
lastModified=AuditStamp(
time=1628882022544, actor="urn:li:corpuser:unknown"
),
Expand Down

0 comments on commit 82bd3c2

Please sign in to comment.