From 3d9ec0f486bece500db2c2446f089cb76f1998ee Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Mon, 25 Nov 2024 11:16:21 +0530 Subject: [PATCH 1/7] fix(ingest/sigma): migrate sigma workbooks from container to dashboard --- .../datahub/ingestion/source/sigma/sigma.py | 101 ++-- .../golden_test_platform_instance_ingest.json | 447 +++++++----------- .../sigma/golden_test_sigma_ingest.json | 329 +++++-------- ...est_sigma_ingest_shared_entities_mces.json | 289 ++++------- 4 files changed, 467 insertions(+), 699 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index dd4b65a2cbdf29..66ac0d0dca2b26 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -4,7 +4,12 @@ import datahub.emitter.mce_builder as builder from datahub.configuration.common import ConfigurationError from datahub.emitter.mcp import MetadataChangeProposalWrapper -from datahub.emitter.mcp_builder import add_entity_to_container, gen_containers +from datahub.emitter.mcp_builder import ( + add_entity_to_container, + add_owner_to_entity_wu, + add_tags_to_entity_wu, + gen_containers, +) from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SourceCapability, @@ -65,6 +70,7 @@ ChartInfoClass, DashboardInfoClass, DataPlatformInstanceClass, + EdgeClass, GlobalTagsClass, InputFieldClass, InputFieldsClass, @@ -74,6 +80,7 @@ SchemaFieldClass, SchemaFieldDataTypeClass, StringTypeClass, + SubTypesClass, TagAssociationClass, ) from datahub.sql_parsing.sqlglot_lineage import create_lineage_sql_parsed_result @@ -505,12 +512,6 @@ def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: yield self._gen_dashboard_info_workunit(page) - yield from add_entity_to_container( - container_key=self._gen_workbook_key(workbook.workbookId), - entity_type="dashboard", - entity_urn=dashboard_urn, - ) - dpi_aspect = self._gen_dataplatform_instance_aspect(dashboard_urn) if dpi_aspect: yield dpi_aspect @@ -531,40 +532,84 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni Map Sigma Workbook to Datahub container """ owner_username = self.sigma_api.get_user_name(workbook.createdBy) - workbook_key = self._gen_workbook_key(workbook.workbookId) - yield from gen_containers( - container_key=workbook_key, - name=workbook.name, - sub_types=[BIContainerSubTypes.SIGMA_WORKBOOK], - parent_container_key=( - self._gen_workspace_key(workbook.workspaceId) - if workbook.workspaceId - else None - ), - extra_properties={ + + dashboard_urn = self._gen_dashboard_urn(workbook.workbookId) + + yield self._gen_entity_status_aspect(dashboard_urn) + + dashboard_info_cls = DashboardInfoClass( + title=workbook.name, + description="", + dashboards=[ + EdgeClass( + destinationUrn=self._gen_dashboard_urn(page.get_urn_part()), + sourceUrn=dashboard_urn, + ) + for page in workbook.pages + ], + charts=[ + builder.make_chart_urn( + platform=self.platform, + platform_instance=self.config.platform_instance, + name=element.get_urn_part(), + ) + for page in workbook.pages + for element in page.elements + ], + externalUrl=workbook.url, + lastModified=ChangeAuditStampsClass(), + customProperties={ "path": workbook.path, "latestVersion": str(workbook.latestVersion), }, - owner_urn=( - builder.make_user_urn(owner_username) - if self.config.ingest_owner and owner_username - else None - ), - external_url=workbook.url, - tags=[workbook.badge] if workbook.badge else None, - created=int(workbook.createdAt.timestamp() * 1000), - last_modified=int(workbook.updatedAt.timestamp() * 1000), ) + yield MetadataChangeProposalWrapper( + entityUrn=dashboard_urn, aspect=dashboard_info_cls + ).as_workunit() + + # Set subtype + yield MetadataChangeProposalWrapper( + entityUrn=f"{dashboard_urn}", + aspect=SubTypesClass(typeNames=[BIContainerSubTypes.SIGMA_WORKBOOK]), + ).as_workunit() + + # Ownership + owner_urn = ( + builder.make_user_urn(owner_username) + if self.config.ingest_owner and owner_username + else None + ) + if owner_urn: + yield from add_owner_to_entity_wu( + entity_type="dashboard", + entity_urn=dashboard_urn, + owner_urn=owner_urn, + ) + + # Tags + tags = [workbook.badge] if workbook.badge else None + if tags: + yield from add_tags_to_entity_wu( + entity_type="dashboard", + entity_urn=dashboard_urn, + tags=sorted(tags), + ) paths = workbook.path.split("/")[1:] if len(paths) > 0 and workbook.workspaceId: yield self._gen_entity_browsepath_aspect( - entity_urn=builder.make_container_urn(workbook_key), + entity_urn=dashboard_urn, parent_entity_urn=builder.make_container_urn( self._gen_workspace_key(workbook.workspaceId) ), paths=paths, ) + elif workbook.workspaceId: + yield from add_entity_to_container( + container_key=self._gen_workspace_key(workbook.workspaceId), + entity_type="dashboard", + entity_urn=dashboard_urn, + ) yield from self._gen_pages_workunit(workbook) diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json index 12bb7734f30a63..d44c3fdbdfe340 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json @@ -321,6 +321,22 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1732513099889, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", @@ -339,6 +355,118 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1732513099890, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "path": "Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "title": "Acryl Workbook", + "description": "", + "charts": [ + "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", + "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" + ], + "datasets": [], + "dashboards": [ + { + "sourceUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)" + }, + { + "sourceUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)" + } + ], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1732513099889, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1732513099890, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "globalTags", + "aspect": { + "json": { + "tags": [ + { + "tag": "urn:li:tag:Warning" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732513099890, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", @@ -401,6 +529,47 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + }, + "systemMetadata": { + "lastObserved": 1732513099891, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732513099891, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "container", "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", @@ -637,10 +806,6 @@ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, { "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" @@ -649,33 +814,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496058, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496019, + "lastObserved": 1732513099908, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -713,6 +852,7 @@ "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -731,86 +871,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "sigma", - "instance": "cloud_instance", - "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", - "path": "Acryl Data", - "latestVersion": "2" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", - "name": "Acryl Workbook", - "created": { - "time": 1713188691477 - }, - "lastModified": { - "time": 1713189117302 - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496016, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - }, - "systemMetadata": { - "lastObserved": 1713794496023, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "subTypes", - "aspect": { - "json": { - "typeNames": [ - "Sigma Workbook" - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496018, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496017, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", @@ -828,72 +888,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496024, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Warning" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496019, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - } - }, - "systemMetadata": { - "lastObserved": 1713794496018, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", @@ -910,31 +904,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496020, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", @@ -1107,10 +1076,6 @@ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, { "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" @@ -1119,7 +1084,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496119, + "lastObserved": 1732513099934, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1156,6 +1121,7 @@ "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -1174,38 +1140,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - } - }, - "systemMetadata": { - "lastObserved": 1713794496020, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - }, - "systemMetadata": { - "lastObserved": 1713794496125, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", @@ -1279,35 +1213,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713794496126, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", @@ -1715,10 +1620,6 @@ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, { "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" @@ -1727,7 +1628,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496194, + "lastObserved": 1732513099969, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json index f800cb19f88115..93ab66af3c6a8c 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json @@ -261,37 +261,8 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "sigma", - "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", - "path": "Acryl Data", - "latestVersion": "2" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", - "name": "Acryl Workbook", - "created": { - "time": 1713188691477 - }, - "lastModified": { - "time": 1713189117302 - } - } - }, - "systemMetadata": { - "lastObserved": 1713795619227, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -300,7 +271,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619228, + "lastObserved": 1732513099680, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -338,32 +309,74 @@ } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "subTypes", "aspect": { "json": { - "path": [ + "typeNames": [ + "Sigma Workbook" + ] + } + }, + "systemMetadata": { + "lastObserved": 1732513099681, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "path": "Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "title": "Acryl Workbook", + "description": "", + "charts": [ + "urn:li:chart:(sigma,kH0MeihtGs)", + "urn:li:chart:(sigma,Ml9C5ezT5W)", + "urn:li:chart:(sigma,tQJu5N1l81)" + ], + "datasets": [], + "dashboards": [ { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)" }, { - "id": "New Folder" + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)" } - ] + ], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } }, "systemMetadata": { - "lastObserved": 1713795619226, + "lastObserved": 1732513099680, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -382,41 +395,51 @@ } }, "systemMetadata": { - "lastObserved": 1713795619229, + "lastObserved": 1732513099681, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "globalTags", "aspect": { "json": { - "typeNames": [ - "Sigma Workbook" + "tags": [ + { + "tag": "urn:li:tag:Warning" + } ] } }, "systemMetadata": { - "lastObserved": 1713795619229, + "lastObserved": 1732513099682, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "browsePathsV2", "aspect": { "json": { - "platform": "urn:li:dataPlatform:sigma" + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "New Folder" + } + ] } }, "systemMetadata": { - "lastObserved": 1713795619228, + "lastObserved": 1713795619226, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -453,6 +476,43 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + }, + "systemMetadata": { + "lastObserved": 1732513099682, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732513099682, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", @@ -470,6 +530,7 @@ "urn:li:chart:(sigma,Ml9C5ezT5W)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -514,42 +575,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "globalTags", - "aspect": { - "json": { - "tags": [ - { - "tag": "urn:li:tag:Warning" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713795619230, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619234, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", @@ -570,47 +595,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - }, - { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713795619234, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - } - }, - "systemMetadata": { - "lastObserved": 1713795619231, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,5LqGLu14qUnqh3cN6wRJBd,PROD)", @@ -637,10 +621,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - }, { "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" @@ -649,28 +629,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619375, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713795619231, + "lastObserved": 1732513099741, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -883,6 +842,7 @@ "urn:li:chart:(sigma,tQJu5N1l81)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -909,10 +869,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - }, { "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" @@ -921,7 +877,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619270, + "lastObserved": 1732513099708, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1179,22 +1135,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619382, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -1409,31 +1349,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - }, - { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1713795619383, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -1549,10 +1464,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", - "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" - }, { "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" @@ -1561,7 +1472,7 @@ } }, "systemMetadata": { - "lastObserved": 1713795619453, + "lastObserved": 1732513099778, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json index d6b702bdfd6695..73627685462dff 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json @@ -279,37 +279,24 @@ } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "containerProperties", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "platform": "sigma", - "workbookId": "9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b", - "path": "New Acryl Data", - "latestVersion": "2" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", - "name": "Acryl Workbook", - "created": { - "time": 1713188691477 - }, - "lastModified": { - "time": 1713189117302 - } + "removed": false } }, "systemMetadata": { - "lastObserved": 1718004101680, + "lastObserved": 1732513100094, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -318,48 +305,80 @@ } }, "systemMetadata": { - "lastObserved": 1718004101680, + "lastObserved": 1718004101684, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "subTypes", "aspect": { "json": { - "platform": "urn:li:dataPlatform:sigma" + "typeNames": [ + "Sigma Workbook" + ] } }, "systemMetadata": { - "lastObserved": 1718004101681, + "lastObserved": 1732513100095, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dashboardInfo", "aspect": { "json": { - "typeNames": [ - "Sigma Workbook" - ] + "customProperties": { + "path": "New Acryl Data", + "latestVersion": "2" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", + "title": "Acryl Workbook", + "description": "", + "charts": [ + "urn:li:chart:(sigma,kH0MeihtGs)", + "urn:li:chart:(sigma,Ml9C5ezT5W)", + "urn:li:chart:(sigma,tQJu5N1l81)" + ], + "datasets": [], + "dashboards": [ + { + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)" + }, + { + "sourceUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "destinationUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)" + } + ], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } } }, "systemMetadata": { - "lastObserved": 1718004101681, + "lastObserved": 1732513100095, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "ownership", "aspect": { @@ -378,14 +397,14 @@ } }, "systemMetadata": { - "lastObserved": 1718004101682, + "lastObserved": 1732513100096, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", "aspectName": "globalTags", "aspect": { @@ -398,60 +417,7 @@ } }, "systemMetadata": { - "lastObserved": 1718004101683, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:b83da80a4d444484521d9f7aca958742" - } - }, - "systemMetadata": { - "lastObserved": 1718004101683, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718004101684, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1718004101684, + "lastObserved": 1732513100096, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -473,6 +439,7 @@ "urn:li:chart:(sigma,Ml9C5ezT5W)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -491,47 +458,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1718004101686, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - }, - { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718004101686, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -581,6 +507,43 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:b83da80a4d444484521d9f7aca958742" + } + }, + "systemMetadata": { + "lastObserved": 1732513100096, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732513100097, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -712,10 +675,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - }, { "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" @@ -724,7 +683,7 @@ } }, "systemMetadata": { - "lastObserved": 1718004101692, + "lastObserved": 1732513100101, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -909,10 +868,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - }, { "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" @@ -921,7 +876,7 @@ } }, "systemMetadata": { - "lastObserved": 1718004101697, + "lastObserved": 1732513100104, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1155,6 +1110,7 @@ "urn:li:chart:(sigma,tQJu5N1l81)" ], "datasets": [], + "dashboards": [], "lastModified": { "created": { "time": 0, @@ -1173,47 +1129,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1718004101704, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - }, - { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718004101704, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", @@ -1469,10 +1384,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - }, { "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" @@ -1481,7 +1392,7 @@ } }, "systemMetadata": { - "lastObserved": 1718004101712, + "lastObserved": 1732513100111, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } From 50feb5f76d3e080e5892c2c7e6db9b2398a7d5f8 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Mon, 25 Nov 2024 17:15:32 +0530 Subject: [PATCH 2/7] fix: PR comments --- .../docs/sources/sigma/sigma_pre.md | 2 +- .../datahub/ingestion/source/sigma/sigma.py | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/docs/sources/sigma/sigma_pre.md b/metadata-ingestion/docs/sources/sigma/sigma_pre.md index 382a2fe67b944d..433f85a69f907c 100644 --- a/metadata-ingestion/docs/sources/sigma/sigma_pre.md +++ b/metadata-ingestion/docs/sources/sigma/sigma_pre.md @@ -16,7 +16,7 @@ This source extracts the following: | Sigma | Datahub | Notes | |------------------------|---------------------------------------------------------------|----------------------------------| | `Workspace` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workspace"` | -| `Workbook` | [Container](../../metamodel/entities/container.md) | SubType `"Sigma Workbook"` | +| `Workbook` | [Dashboard](../../metamodel/entities/dashboard.md) | SubType `"Sigma Workbook"` | | `Page` | [Dashboard](../../metamodel/entities/dashboard.md) | | | `Element` | [Chart](../../metamodel/entities/chart.md) | | | `Dataset` | [Dataset](../../metamodel/entities/dataset.md) | SubType `"Sigma Dataset"` | diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index 66ac0d0dca2b26..31fdbeac16a0fa 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -64,6 +64,7 @@ UpstreamLineage, ) from datahub.metadata.schema_classes import ( + AuditStampClass, BrowsePathEntryClass, BrowsePathsV2Class, ChangeAuditStampsClass, @@ -537,6 +538,15 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni yield self._gen_entity_status_aspect(dashboard_urn) + lastModified = AuditStampClass( + time=int(workbook.updatedAt.timestamp() * 1000), + actor="urn:li:corpuser:datahub", + ) + created = AuditStampClass( + time=int(workbook.createdAt.timestamp() * 1000), + actor="urn:li:corpuser:datahub", + ) + dashboard_info_cls = DashboardInfoClass( title=workbook.name, description="", @@ -547,17 +557,10 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni ) for page in workbook.pages ], - charts=[ - builder.make_chart_urn( - platform=self.platform, - platform_instance=self.config.platform_instance, - name=element.get_urn_part(), - ) - for page in workbook.pages - for element in page.elements - ], externalUrl=workbook.url, - lastModified=ChangeAuditStampsClass(), + lastModified=ChangeAuditStampsClass( + created=created, lastModified=lastModified + ), customProperties={ "path": workbook.path, "latestVersion": str(workbook.latestVersion), @@ -569,7 +572,7 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni # Set subtype yield MetadataChangeProposalWrapper( - entityUrn=f"{dashboard_urn}", + entityUrn=dashboard_urn, aspect=SubTypesClass(typeNames=[BIContainerSubTypes.SIGMA_WORKBOOK]), ).as_workunit() From 516fb33a2b90d291909481ce9b73a25de56c51e6 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Mon, 25 Nov 2024 17:16:14 +0530 Subject: [PATCH 3/7] test: update golden file for latest changes --- .../golden_test_platform_instance_ingest.json | 16 ++++++---------- .../sigma/golden_test_sigma_ingest.json | 16 ++++++---------- ...n_test_sigma_ingest_shared_entities_mces.json | 16 ++++++---------- 3 files changed, 18 insertions(+), 30 deletions(-) diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json index d44c3fdbdfe340..ceea74b7158d71 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json @@ -387,11 +387,7 @@ "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", "title": "Acryl Workbook", "description": "", - "charts": [ - "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", - "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" - ], + "charts": [], "datasets": [], "dashboards": [ { @@ -405,18 +401,18 @@ ], "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1713188691477, + "actor": "urn:li:corpuser:datahub" }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1713189117302, + "actor": "urn:li:corpuser:datahub" } } } }, "systemMetadata": { - "lastObserved": 1732513099889, + "lastObserved": 1732535136125, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json index 93ab66af3c6a8c..8820164520402e 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json @@ -340,11 +340,7 @@ "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", "title": "Acryl Workbook", "description": "", - "charts": [ - "urn:li:chart:(sigma,kH0MeihtGs)", - "urn:li:chart:(sigma,Ml9C5ezT5W)", - "urn:li:chart:(sigma,tQJu5N1l81)" - ], + "charts": [], "datasets": [], "dashboards": [ { @@ -358,18 +354,18 @@ ], "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1713188691477, + "actor": "urn:li:corpuser:datahub" }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1713189117302, + "actor": "urn:li:corpuser:datahub" } } } }, "systemMetadata": { - "lastObserved": 1732513099680, + "lastObserved": 1732535135915, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json index 73627685462dff..14be4c67d4b609 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json @@ -342,11 +342,7 @@ "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7", "title": "Acryl Workbook", "description": "", - "charts": [ - "urn:li:chart:(sigma,kH0MeihtGs)", - "urn:li:chart:(sigma,Ml9C5ezT5W)", - "urn:li:chart:(sigma,tQJu5N1l81)" - ], + "charts": [], "datasets": [], "dashboards": [ { @@ -360,18 +356,18 @@ ], "lastModified": { "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1713188691477, + "actor": "urn:li:corpuser:datahub" }, "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" + "time": 1713189117302, + "actor": "urn:li:corpuser:datahub" } } } }, "systemMetadata": { - "lastObserved": 1732513100095, + "lastObserved": 1732535136409, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } From e72f70bf321390266340154354ccf3fc0f363631 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Mon, 25 Nov 2024 20:14:03 +0530 Subject: [PATCH 4/7] fix: minor changes --- .../ingestion/source/sigma/data_classes.py | 1 + .../datahub/ingestion/source/sigma/sigma.py | 51 ++++++++++++------- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py index 922b0be3b4a93c..5a657d804cb7bf 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/data_classes.py @@ -80,6 +80,7 @@ class Workbook(BaseModel): path: str latestVersion: int workspaceId: Optional[str] = None + description: Optional[str] = None pages: List[Page] = [] badge: Optional[str] = None diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index 31fdbeac16a0fa..560409759e0de7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -432,11 +432,11 @@ def _gen_elements_workunit( elements: List[Element], workbook: Workbook, all_input_fields: List[InputFieldClass], + paths: List[str], ) -> Iterable[MetadataWorkUnit]: """ Map Sigma page element to Datahub Chart """ - for element in elements: chart_urn = builder.make_chart_urn( platform=self.platform, @@ -467,11 +467,14 @@ def _gen_elements_workunit( ), ).as_workunit() - yield from add_entity_to_container( - container_key=self._gen_workbook_key(workbook.workbookId), - entity_type="chart", - entity_urn=chart_urn, - ) + if workbook.workspaceId: + yield self._gen_entity_browsepath_aspect( + entity_urn=chart_urn, + parent_entity_urn=builder.make_container_urn( + self._gen_workspace_key(workbook.workspaceId) + ), + paths=paths + [workbook.name], + ) # Add sigma dataset's upstream dataset urn mapping for dataset_urn, upstream_dataset_urns in inputs.items(): @@ -502,7 +505,9 @@ def _gen_elements_workunit( all_input_fields.extend(element_input_fields) - def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: + def _gen_pages_workunit( + self, workbook: Workbook, paths: List[str] + ) -> Iterable[MetadataWorkUnit]: """ Map Sigma workbook page to Datahub dashboard """ @@ -519,8 +524,17 @@ def _gen_pages_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUnit]: all_input_fields: List[InputFieldClass] = [] + if workbook.workspaceId: + yield self._gen_entity_browsepath_aspect( + entity_urn=dashboard_urn, + parent_entity_urn=builder.make_container_urn( + self._gen_workspace_key(workbook.workspaceId) + ), + paths=paths + [workbook.name], + ) + yield from self._gen_elements_workunit( - page.elements, workbook, all_input_fields + page.elements, workbook, all_input_fields, paths ) yield MetadataChangeProposalWrapper( @@ -549,7 +563,7 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni dashboard_info_cls = DashboardInfoClass( title=workbook.name, - description="", + description=workbook.description if workbook.description else "", dashboards=[ EdgeClass( destinationUrn=self._gen_dashboard_urn(page.get_urn_part()), @@ -599,22 +613,23 @@ def _gen_workbook_workunit(self, workbook: Workbook) -> Iterable[MetadataWorkUni ) paths = workbook.path.split("/")[1:] - if len(paths) > 0 and workbook.workspaceId: + if workbook.workspaceId: yield self._gen_entity_browsepath_aspect( entity_urn=dashboard_urn, parent_entity_urn=builder.make_container_urn( self._gen_workspace_key(workbook.workspaceId) ), - paths=paths, - ) - elif workbook.workspaceId: - yield from add_entity_to_container( - container_key=self._gen_workspace_key(workbook.workspaceId), - entity_type="dashboard", - entity_urn=dashboard_urn, + paths=paths + [workbook.name], ) - yield from self._gen_pages_workunit(workbook) + if len(paths) == 0: + yield from add_entity_to_container( + container_key=self._gen_workspace_key(workbook.workspaceId), + entity_type="dashboard", + entity_urn=dashboard_urn, + ) + + yield from self._gen_pages_workunit(workbook, paths) def _gen_sigma_dataset_upstream_lineage_workunit( self, From fa325006e9b243e8673433156b1dafcbdd6de5cc Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Mon, 25 Nov 2024 20:15:53 +0530 Subject: [PATCH 5/7] test: update golden file for latest changes --- .../golden_test_platform_instance_ingest.json | 160 +++++++++++------- .../sigma/golden_test_sigma_ingest.json | 128 +++++++------- ...est_sigma_ingest_shared_entities_mces.json | 128 +++++++------- 3 files changed, 242 insertions(+), 174 deletions(-) diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json index ceea74b7158d71..82a8fcbf002c2c 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json @@ -541,6 +541,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545849038, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", @@ -549,6 +581,10 @@ "aspect": { "json": { "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" @@ -556,12 +592,15 @@ { "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099891, + "lastObserved": 1732545849036, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -630,22 +669,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - }, - "systemMetadata": { - "lastObserved": 1713794496055, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", @@ -803,14 +826,21 @@ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" }, { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099908, + "lastObserved": 1732545849054, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -900,22 +930,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - }, - "systemMetadata": { - "lastObserved": 1713794496116, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", @@ -1060,6 +1074,38 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545849086, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", @@ -1073,14 +1119,21 @@ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" }, { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099934, + "lastObserved": 1732545849082, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1209,22 +1262,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" - } - }, - "systemMetadata": { - "lastObserved": 1713794496189, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", @@ -1617,14 +1654,21 @@ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" }, { - "id": "urn:li:container:084a2e283eddfc576ce70989b395a7d8", - "urn": "urn:li:container:084a2e283eddfc576ce70989b395a7d8" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099969, + "lastObserved": 1732545849117, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json index 8820164520402e..bb37e7029330b2 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest.json @@ -472,6 +472,30 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545848809, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", @@ -499,12 +523,15 @@ { "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099682, + "lastObserved": 1732545848807, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -618,30 +645,17 @@ "json": { "path": [ { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099741, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619373, + "lastObserved": 1732545848872, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -857,6 +871,30 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545848877, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", @@ -866,14 +904,17 @@ "json": { "path": [ { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099708, + "lastObserved": 1732545848829, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1131,38 +1172,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619267, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1713795619449, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", @@ -1461,14 +1470,17 @@ "json": { "path": [ { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f", + "urn": "urn:li:container:46c912b7a3f62c8e3269e559648c4b2f" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513099778, + "lastObserved": 1732545848921, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json index 14be4c67d4b609..1ce671f09d7765 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_sigma_ingest_shared_entities_mces.json @@ -505,16 +505,24 @@ }, { "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "entityUrn": "urn:li:dashboard:(sigma,OSnGLBzL1i)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:b83da80a4d444484521d9f7aca958742" + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" + } + ] } }, "systemMetadata": { - "lastObserved": 1732513100096, + "lastObserved": 1732545849249, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -523,35 +531,38 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "container", "aspect": { "json": { - "path": [ - { - "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", - "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" - } - ] + "container": "urn:li:container:b83da80a4d444484521d9f7aca958742" } }, "systemMetadata": { - "lastObserved": 1732513100097, + "lastObserved": 1732513100096, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,kH0MeihtGs)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "browsePathsV2", "aspect": { "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" + } + ] } }, "systemMetadata": { - "lastObserved": 1718004101689, + "lastObserved": 1732545849248, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -672,14 +683,17 @@ "json": { "path": [ { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513100101, + "lastObserved": 1732545849252, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -733,22 +747,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1718004101695, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,Ml9C5ezT5W)", @@ -865,14 +863,17 @@ "json": { "path": [ { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513100104, + "lastObserved": 1732545849255, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1141,6 +1142,30 @@ "lastRunId": "no-run-id-provided" } }, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732545849260, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", @@ -1174,22 +1199,6 @@ "lastRunId": "no-run-id-provided" } }, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" - } - }, - "systemMetadata": { - "lastObserved": 1718004101708, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, { "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,tQJu5N1l81)", @@ -1381,14 +1390,17 @@ "json": { "path": [ { - "id": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02", - "urn": "urn:li:container:140db5e9decc9b6ec67fa1e8207b6f02" + "id": "urn:li:container:b83da80a4d444484521d9f7aca958742", + "urn": "urn:li:container:b83da80a4d444484521d9f7aca958742" + }, + { + "id": "Acryl Workbook" } ] } }, "systemMetadata": { - "lastObserved": 1732513100111, + "lastObserved": 1732545849264, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } From 2179198b057332bc078409b8a4bb4b3a7e7a949c Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Tue, 26 Nov 2024 01:00:27 +0530 Subject: [PATCH 6/7] fix: PR comment --- .../src/datahub/ingestion/source/sigma/sigma.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py index 560409759e0de7..e96eeb58d96efe 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py @@ -265,11 +265,6 @@ def _gen_entity_browsepath_aspect( entries = [ BrowsePathEntryClass(id=parent_entity_urn, urn=parent_entity_urn) ] + [BrowsePathEntryClass(id=path) for path in paths] - if self.config.platform_instance: - urn = builder.make_dataplatform_instance_urn( - self.platform, self.config.platform_instance - ) - entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries return MetadataChangeProposalWrapper( entityUrn=entity_urn, aspect=BrowsePathsV2Class(entries), From 4d343be3c621cbf075c78f0e2b86c75c9cd81940 Mon Sep 17 00:00:00 2001 From: sagar-salvi-apptware Date: Tue, 26 Nov 2024 13:46:18 +0530 Subject: [PATCH 7/7] test: update golden files --- .../golden_test_platform_instance_ingest.json | 1254 ++++++++--------- 1 file changed, 613 insertions(+), 641 deletions(-) diff --git a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json index 82a8fcbf002c2c..645e710309b0da 100644 --- a/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json +++ b/metadata-ingestion/tests/integration/sigma/golden_test_platform_instance_ingest.json @@ -10,23 +10,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496006, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", - "changeType": "UPSERT", - "aspectName": "container", - "aspect": { - "json": { - "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - } - }, - "systemMetadata": { - "lastObserved": 1713794496008, + "lastObserved": 1732608523763, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -56,7 +40,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496007, + "lastObserved": 1732608523764, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -65,40 +49,30 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", + "aspectName": "container", "aspect": { "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" } }, "systemMetadata": { - "lastObserved": 1713794496008, + "lastObserved": 1732608523764, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - } - ] + "removed": false } }, "systemMetadata": { - "lastObserved": 1713794496010, + "lastObserved": 1732608523833, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -107,16 +81,15 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "typeNames": [ - "Sigma Dataset" - ] + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" } }, "systemMetadata": { - "lastObserved": 1713794496009, + "lastObserved": 1732608523764, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -125,40 +98,44 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "subTypes", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "typeNames": [ + "Sigma Dataset" + ] } }, "systemMetadata": { - "lastObserved": 1713794496009, + "lastObserved": 1732608523765, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "browsePathsV2", "aspect": { "json": { - "removed": false + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" + } + ] } }, "systemMetadata": { - "lastObserved": 1713794496011, + "lastObserved": 1732608523835, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -167,14 +144,14 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "status", "aspect": { "json": { - "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + "removed": false } }, "systemMetadata": { - "lastObserved": 1713794496012, + "lastObserved": 1732608523781, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -206,33 +183,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496012, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "changeType": "UPSERT", - "aspectName": "ownership", - "aspect": { - "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496201, + "lastObserved": 1732608523781, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -241,49 +192,26 @@ "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - } - }, - "systemMetadata": { - "lastObserved": 1713794496013, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "container", "aspect": { "json": { - "typeNames": [ - "Sigma Workspace" - ] + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" } }, "systemMetadata": { - "lastObserved": 1713794496200, + "lastObserved": 1732608523782, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", "changeType": "UPSERT", "aspectName": "browsePathsV2", "aspect": { "json": { "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" @@ -291,22 +219,19 @@ { "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "New Folder" } ] } }, "systemMetadata": { - "lastObserved": 1713794496015, + "lastObserved": 1732608523765, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -316,41 +241,88 @@ } }, "systemMetadata": { - "lastObserved": 1713794496200, + "lastObserved": 1732608523782, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "subTypes", "aspect": { "json": { - "removed": false + "typeNames": [ + "Sigma Dataset" + ] } }, "systemMetadata": { - "lastObserved": 1732513099889, + "lastObserved": 1732608523783, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "chartInfo", "aspect": { "json": { - "typeNames": [ - "Sigma Dataset" + "customProperties": { + "VizualizationType": "bar", + "type": "visualization" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", + "title": "Count of Profile Id by Status", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" + } ] } }, "systemMetadata": { - "lastObserved": 1713794496014, + "lastObserved": 1732608523833, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523765, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -359,16 +331,14 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "subTypes", + "aspectName": "status", "aspect": { "json": { - "typeNames": [ - "Sigma Workbook" - ] + "removed": false } }, "systemMetadata": { - "lastObserved": 1732513099890, + "lastObserved": 1732608523784, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -412,7 +382,7 @@ } }, "systemMetadata": { - "lastObserved": 1732535136125, + "lastObserved": 1732608523785, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -421,97 +391,113 @@ "entityType": "dashboard", "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "subTypes", "aspect": { "json": { - "owners": [ - { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" - } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + "typeNames": [ + "Sigma Workbook" + ] } }, "systemMetadata": { - "lastObserved": 1732513099890, + "lastObserved": 1732608523785, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "browsePathsV2", "aspect": { "json": { - "tags": [ + "path": [ { - "tag": "urn:li:tag:Warning" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "New Folder" } ] } }, "systemMetadata": { - "lastObserved": 1732513099890, + "lastObserved": 1732608523783, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "globalTags", + "aspectName": "ownership", "aspect": { "json": { - "tags": [ + "owners": [ { - "tag": "urn:li:tag:Deprecated" + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" } - ] + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { - "lastObserved": 1713794496015, + "lastObserved": 1732608523786, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", "changeType": "UPSERT", - "aspectName": "ownership", + "aspectName": "globalTags", "aspect": { "json": { - "owners": [ + "tags": [ { - "owner": "urn:li:corpuser:Shubham_Jagtap", - "type": "DATAOWNER" + "tag": "urn:li:tag:Warning" } - ], - "ownerTypes": {}, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } + ] } }, "systemMetadata": { - "lastObserved": 1713794496014, + "lastObserved": 1732608523786, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "changeType": "UPSERT", + "aspectName": "container", + "aspect": { + "json": { + "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + } + }, + "systemMetadata": { + "lastObserved": 1732608523786, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -520,23 +506,60 @@ } }, "systemMetadata": { - "lastObserved": 1713794496199, + "lastObserved": 1732608523787, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.9bbbe3b0-c0c8-4fac-b6f1-8dfebfe74f8b)", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", "changeType": "UPSERT", - "aspectName": "container", + "aspectName": "dashboardInfo", "aspect": { "json": { - "container": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + "customProperties": { + "ElementsCount": "2" + }, + "title": "Page 1", + "description": "", + "charts": [ + "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523788, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" } }, "systemMetadata": { - "lastObserved": 1732513099891, + "lastObserved": 1732608523788, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -549,10 +572,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" @@ -568,7 +587,7 @@ } }, "systemMetadata": { - "lastObserved": 1732545849038, + "lastObserved": 1732608523788, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -581,10 +600,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" @@ -600,85 +615,67 @@ } }, "systemMetadata": { - "lastObserved": 1732545849036, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "changeType": "UPSERT", - "aspectName": "containerProperties", - "aspect": { - "json": { - "customProperties": { - "platform": "sigma", - "instance": "cloud_instance", - "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" - }, - "name": "Acryl Data", - "created": { - "time": 1710232264826 - }, - "lastModified": { - "time": 1710232264826 - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496199, + "lastObserved": 1732608523787, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "status", + "aspectName": "globalTags", "aspect": { "json": { - "removed": false + "tags": [ + { + "tag": "urn:li:tag:Deprecated" + } + ] } }, "systemMetadata": { - "lastObserved": 1713794496053, + "lastObserved": 1732608523783, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "container", - "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.5LqGLu14qUnqh3cN6wRJBd,PROD)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "ownership", "aspect": { "json": { - "path": [ + "owners": [ { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" } - ] + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } } }, "systemMetadata": { - "lastObserved": 1713794496202, + "lastObserved": 1732608523782, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", "changeType": "UPSERT", "aspectName": "inputFields", "aspect": { "json": { "fields": [ { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Pk)", "schemaField": { "fieldPath": "Pk", "nullable": false, @@ -693,9 +690,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Profile Id)", "schemaField": { - "fieldPath": "Pet Fk", + "fieldPath": "Profile Id", "nullable": false, "type": { "type": { @@ -708,9 +705,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Status)", "schemaField": { - "fieldPath": "Human Fk", + "fieldPath": "Status", "nullable": false, "type": { "type": { @@ -723,9 +720,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Created At)", "schemaField": { - "fieldPath": "Status", + "fieldPath": "Created At", "nullable": false, "type": { "type": { @@ -738,9 +735,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", "schemaField": { - "fieldPath": "Created At", + "fieldPath": "Updated At", "nullable": false, "type": { "type": { @@ -753,9 +750,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", "schemaField": { - "fieldPath": "Updated At", + "fieldPath": "Count of Profile Id", "nullable": false, "type": { "type": { @@ -771,7 +768,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496055, + "lastObserved": 1732608523834, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -780,35 +777,14 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", "changeType": "UPSERT", - "aspectName": "chartInfo", + "aspectName": "status", "aspect": { "json": { - "customProperties": { - "VizualizationType": "levelTable", - "type": "table" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", - "title": "ADOPTIONS", - "description": "", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" - } - ] + "removed": false } }, "systemMetadata": { - "lastObserved": 1718348049212, + "lastObserved": 1732608523803, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -821,10 +797,6 @@ "aspect": { "json": { "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, { "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" @@ -840,45 +812,25 @@ } }, "systemMetadata": { - "lastObserved": 1732545849054, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496021, + "lastObserved": 1732608523806, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", "changeType": "UPSERT", - "aspectName": "dashboardInfo", + "aspectName": "chartInfo", "aspect": { "json": { "customProperties": { - "ElementsCount": "2" + "VizualizationType": "levelTable", + "type": "table" }, - "title": "Page 1", + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=kH0MeihtGs&:fullScreen=true", + "title": "ADOPTIONS", "description": "", - "charts": [ - "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", - "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)" - ], - "datasets": [], - "dashboards": [], "lastModified": { "created": { "time": 0, @@ -888,58 +840,30 @@ "time": 0, "actor": "urn:li:corpuser:unknown" } - } + }, + "inputs": [ + { + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" + } + ] } }, "systemMetadata": { - "lastObserved": 1713794496022, + "lastObserved": 1732608523804, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.OSnGLBzL1i)", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.kH0MeihtGs)", "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - } - }, - "systemMetadata": { - "lastObserved": 1713794496023, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496114, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "inputFields", + "aspectName": "inputFields", "aspect": { "json": { "fields": [ { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Pk)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pk)", "schemaField": { "fieldPath": "Pk", "nullable": false, @@ -954,9 +878,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Profile Id)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Pet Fk)", "schemaField": { - "fieldPath": "Profile Id", + "fieldPath": "Pet Fk", "nullable": false, "type": { "type": { @@ -969,9 +893,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Status)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Human Fk)", "schemaField": { - "fieldPath": "Status", + "fieldPath": "Human Fk", "nullable": false, "type": { "type": { @@ -984,9 +908,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Created At)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Status)", "schemaField": { - "fieldPath": "Created At", + "fieldPath": "Status", "nullable": false, "type": { "type": { @@ -999,9 +923,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Created At)", "schemaField": { - "fieldPath": "Updated At", + "fieldPath": "Created At", "nullable": false, "type": { "type": { @@ -1014,9 +938,9 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.kH0MeihtGs),Updated At)", "schemaField": { - "fieldPath": "Count of Profile Id", + "fieldPath": "Updated At", "nullable": false, "type": { "type": { @@ -1032,232 +956,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496117, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "json": { - "customProperties": { - "VizualizationType": "bar", - "type": "visualization" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=Ml9C5ezT5W&:fullScreen=true", - "title": "Count of Profile Id by Status", - "description": "", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718348049268, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "Acryl Workbook" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1732545849086, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W)", - "changeType": "UPSERT", - "aspectName": "browsePathsV2", - "aspect": { - "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" - }, - { - "id": "Acryl Workbook" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1732545849082, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496124, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "dashboardInfo", - "aspect": { - "json": { - "customProperties": { - "ElementsCount": "1" - }, - "title": "Page 2", - "description": "", - "charts": [ - "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" - ], - "datasets": [], - "dashboards": [], - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - } - } - }, - "systemMetadata": { - "lastObserved": 1713794496125, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "chartInfo", - "aspect": { - "json": { - "customProperties": { - "VizualizationType": "levelTable", - "type": "table" - }, - "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", - "title": "PETS ADOPTIONS JOIN", - "description": "", - "lastModified": { - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - } - }, - "inputs": [ - { - "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" - }, - { - "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" - } - ] - } - }, - "systemMetadata": { - "lastObserved": 1718348049351, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "chart", - "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", - "changeType": "UPSERT", - "aspectName": "status", - "aspect": { - "json": { - "removed": false - } - }, - "systemMetadata": { - "lastObserved": 1713794496188, - "runId": "sigma-test", - "lastRunId": "no-run-id-provided" - } -}, -{ - "entityType": "dashboard", - "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "json": { - "platform": "urn:li:dataPlatform:sigma", - "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - } - }, - "systemMetadata": { - "lastObserved": 1713794496126, + "lastObserved": 1732608523804, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1421,40 +1120,219 @@ } }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", - "schemaField": { - "fieldPath": "Updated At", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Updated At)", + "schemaField": { + "fieldPath": "Updated At", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + }, + { + "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", + "schemaField": { + "fieldPath": "Count of Profile Id", + "nullable": false, + "type": { + "type": { + "com.linkedin.schema.StringType": {} + } + }, + "nativeDataType": "String", + "recursive": false, + "isPartOfKey": false + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732608523836, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1732608523838, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dashboardInfo", + "aspect": { + "json": { + "customProperties": { + "ElementsCount": "1" + }, + "title": "Page 2", + "description": "", + "charts": [ + "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)" + ], + "datasets": [], + "dashboards": [], + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523838, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "json": { + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1732608523839, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dashboard", + "entityUrn": "urn:li:dashboard:(sigma,cloud_instance.DFSieiAcgo)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" }, { - "schemaFieldUrn": "urn:li:schemaField:(urn:li:chart:(sigma,cloud_instance.Ml9C5ezT5W),Count of Profile Id)", - "schemaField": { - "fieldPath": "Count of Profile Id", - "nullable": false, - "type": { - "type": { - "com.linkedin.schema.StringType": {} - } - }, - "nativeDataType": "String", - "recursive": false, - "isPartOfKey": false - } + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732608523839, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "json": { + "upstreams": [ + { + "auditStamp": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)", + "type": "COPY" } ] } }, "systemMetadata": { - "lastObserved": 1713794496120, + "lastObserved": 1732608523874, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Deprecated", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Deprecated" + } + }, + "systemMetadata": { + "lastObserved": 1732608523874, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "json": { + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1732608523872, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "ownership", + "aspect": { + "json": { + "owners": [ + { + "owner": "urn:li:corpuser:Shubham_Jagtap", + "type": "DATAOWNER" + } + ], + "ownerTypes": {}, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523873, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1636,7 +1514,7 @@ } }, "systemMetadata": { - "lastObserved": 1713794496194, + "lastObserved": 1732608523870, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1645,30 +1523,54 @@ "entityType": "chart", "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", "changeType": "UPSERT", - "aspectName": "browsePathsV2", + "aspectName": "status", "aspect": { "json": { - "path": [ - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" - }, - { - "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", - "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + "removed": false + } + }, + "systemMetadata": { + "lastObserved": 1732608523866, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", + "changeType": "UPSERT", + "aspectName": "chartInfo", + "aspect": { + "json": { + "customProperties": { + "VizualizationType": "levelTable", + "type": "table" + }, + "externalUrl": "https://app.sigmacomputing.com/acryldata/workbook/4JRFW1HThPI1K3YTjouXI7?:nodeId=tQJu5N1l81&:fullScreen=true", + "title": "PETS ADOPTIONS JOIN", + "description": "", + "lastModified": { + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + }, + "inputs": [ { - "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", - "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + "string": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)" }, { - "id": "Acryl Workbook" + "string": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.adoptions,DEV)" } ] } }, "systemMetadata": { - "lastObserved": 1732545849117, + "lastObserved": 1732608523866, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } @@ -1850,64 +1752,134 @@ } }, "systemMetadata": { - "lastObserved": 1713794496190, + "lastObserved": 1732608523867, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:sigma,cloud_instance.49HFLTr6xytgrPly3PFsNC,PROD)", + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", "changeType": "UPSERT", - "aspectName": "upstreamLineage", + "aspectName": "dataPlatformInstance", "aspect": { "json": { - "upstreams": [ + "platform": "urn:li:dataPlatform:sigma", + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + } + }, + "systemMetadata": { + "lastObserved": 1732608523873, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [ { - "auditStamp": { - "time": 0, - "actor": "urn:li:corpuser:unknown" - }, - "dataset": "urn:li:dataset:(urn:li:dataPlatform:snowflake,dev_instance.long_tail_companions.adoption.pets,DEV)", - "type": "COPY" + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" } ] } }, "systemMetadata": { - "lastObserved": 1718348049380, + "lastObserved": 1732608523874, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { - "entityType": "tag", - "entityUrn": "urn:li:tag:Warning", + "entityType": "chart", + "entityUrn": "urn:li:chart:(sigma,cloud_instance.tQJu5N1l81)", "changeType": "UPSERT", - "aspectName": "tagKey", + "aspectName": "browsePathsV2", "aspect": { "json": { - "name": "Warning" + "path": [ + { + "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)", + "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:sigma,cloud_instance)" + }, + { + "id": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "urn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d" + }, + { + "id": "Acryl Workbook" + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1732608523869, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "containerProperties", + "aspect": { + "json": { + "customProperties": { + "platform": "sigma", + "instance": "cloud_instance", + "workspaceId": "3ee61405-3be2-4000-ba72-60d36757b95b" + }, + "name": "Acryl Data", + "created": { + "time": 1710232264826 + }, + "lastModified": { + "time": 1710232264826 + } + } + }, + "systemMetadata": { + "lastObserved": 1732608523872, + "runId": "sigma-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "container", + "entityUrn": "urn:li:container:abbebb5181bf9ba2d905d2dea7d8704d", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Sigma Workspace" + ] } }, "systemMetadata": { - "lastObserved": 1713794496203, + "lastObserved": 1732608523873, "runId": "sigma-test", "lastRunId": "no-run-id-provided" } }, { "entityType": "tag", - "entityUrn": "urn:li:tag:Deprecated", + "entityUrn": "urn:li:tag:Warning", "changeType": "UPSERT", "aspectName": "tagKey", "aspect": { "json": { - "name": "Deprecated" + "name": "Warning" } }, "systemMetadata": { - "lastObserved": 1713794496203, + "lastObserved": 1732608523875, "runId": "sigma-test", "lastRunId": "no-run-id-provided" }