From cdeee9877bdd395374a5456f9cc4b42cbfa0e9ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Sun, 13 Feb 2022 10:35:28 +0100 Subject: [PATCH 01/12] feat(ingestion): adds platform instance capability to glue connector --- metadata-ingestion/setup.py | 9 +- metadata-ingestion/source_docs/glue.md | 44 +- .../ingestion/source/aws/aws_common.py | 2 +- .../src/datahub/ingestion/source/aws/glue.py | 64 +- .../glue_mces_platform_instance_golden.json | 1082 +++++++++++++++++ .../tests/unit/test_glue_source.py | 69 +- .../tests/unit/test_glue_source_stubs.py | 8 +- 7 files changed, 1227 insertions(+), 51 deletions(-) create mode 100644 metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index a6e6f0369e6391..ec0885ad45146f 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -92,6 +92,11 @@ def get_long_description(): "cryptography", } +trino = { + "trino>=0.308", + "trino[sqlalchemy]>=0.308" +} + # Note: for all of these, framework_common will be added. plugins: Dict[str, Set[str]] = { # Sink plugins. @@ -148,8 +153,8 @@ def get_long_description(): "sqlalchemy": sql_common, "superset": {"requests", "sqlalchemy", "great_expectations"}, "tableau": {"tableauserverclient>=0.17.0"}, - "trino": sql_common | {"trino"}, - "starburst-trino-usage": sql_common | {"trino"}, + "trino": sql_common | trino, + "starburst-trino-usage": sql_common | trino, "nifi": {"requests", "packaging"}, } diff --git a/metadata-ingestion/source_docs/glue.md b/metadata-ingestion/source_docs/glue.md index 27f5074f9775fd..a14ea96036c05c 100644 --- a/metadata-ingestion/source_docs/glue.md +++ b/metadata-ingestion/source_docs/glue.md @@ -19,7 +19,7 @@ This plugin extracts the following: | Capability | Status | Details | | -----------| ------ | ---- | -| Platform Instance | 🛑 | [link](../../docs/platform-instances.md) | +| Platform Instance | ✔ | [link](../../docs/platform-instances.md) | ## Quickstart recipe @@ -73,26 +73,28 @@ plus `s3:GetObject` for the job script locations. Note that a `.` is used to denote nested fields in the YAML recipe. -| Field | Required | Default | Description | -| ------------------------------- | -------- | ------------ | ---------------------------------------------------------------------------------- | -| `aws_region` | ✅ | | AWS region code. | -| `env` | | `"PROD"` | Environment to use in namespace when constructing URNs. | -| `aws_access_key_id` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | -| `aws_secret_access_key` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | -| `aws_session_token` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | -| `aws_role` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | -| `extract_transforms` | | `True` | Whether to extract Glue transform jobs. | -| `database_pattern.allow` | | | List of regex patterns for databases to include in ingestion. | -| `database_pattern.deny` | | | List of regex patterns for databases to exclude from ingestion. | -| `database_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. | -| `table_pattern.allow` | | | List of regex patterns for tables to include in ingestion. | -| `table_pattern.deny` | | | List of regex patterns for tables to exclude from ingestion. | -| `table_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. | -| `underlying_platform` | | `glue` | Override for platform name. Allowed values - `glue`, `athena` | -| `ignore_unsupported_connectors` | | `True` | Whether to ignore unsupported connectors. If disabled, an error will be raised. | -| `emit_s3_lineage` | | `True` | Whether to emit S3-to-Glue lineage. | -| `glue_s3_lineage_direction` | | `upstream` | If `upstream`, S3 is upstream to Glue. If `downstream` S3 is downstream to Glue. | -| `extract_owners` | | `True` | When enabled, extracts ownership from Glue directly and overwrites existing owners. When disabled, ownership is left empty for datasets. | +| Field | Required | Default | Description | +|---------------------------------| -------- | ------------ |------------------------------------------------------------------------------------------------------------------------------------------| +| `aws_region` | ✅ | | AWS region code. | +| `env` | | `"PROD"` | Environment to use in namespace when constructing URNs. | +| `aws_access_key_id` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | +| `aws_secret_access_key` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | +| `aws_session_token` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | +| `aws_role` | | Autodetected | See https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html | +| `extract_transforms` | | `True` | Whether to extract Glue transform jobs. | +| `database_pattern.allow` | | | List of regex patterns for databases to include in ingestion. | +| `database_pattern.deny` | | | List of regex patterns for databases to exclude from ingestion. | +| `database_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. | +| `table_pattern.allow` | | | List of regex patterns for tables to include in ingestion. | +| `table_pattern.deny` | | | List of regex patterns for tables to exclude from ingestion. | +| `table_pattern.ignoreCase` | | `True` | Whether to ignore case sensitivity during pattern matching. | +| `platform` | | `glue` | Override for platform name. Allowed values - `glue`, `athena` | +| `platform_instance` | | None | The Platform instance to use while constructing URNs. | +| `underlying_platform` | | `glue` | @deprecated(Use `platform`) Override for platform name. Allowed values - `glue`, `athena` | +| `ignore_unsupported_connectors` | | `True` | Whether to ignore unsupported connectors. If disabled, an error will be raised. | +| `emit_s3_lineage` | | `True` | Whether to emit S3-to-Glue lineage. | +| `glue_s3_lineage_direction` | | `upstream` | If `upstream`, S3 is upstream to Glue. If `downstream` S3 is downstream to Glue. | +| `extract_owners` | | `True` | When enabled, extracts ownership from Glue directly and overwrites existing owners. When disabled, ownership is left empty for datasets. | ## Compatibility diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py index d12c215bd3ab66..394320bc0e4b23 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/aws_common.py @@ -6,7 +6,7 @@ from datahub.configuration import ConfigModel from datahub.configuration.common import AllowDenyPattern -from datahub.emitter.mce_builder import DEFAULT_ENV +from datahub.configuration.source_common import DEFAULT_ENV if TYPE_CHECKING: diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index 1491992469983d..b14aec05690ff6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -10,6 +10,7 @@ from pydantic import validator from datahub.configuration.common import ConfigurationError +from datahub.configuration.source_common import PlatformSourceConfigBase from datahub.emitter import mce_builder from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.common import PipelineContext @@ -45,8 +46,11 @@ logger = logging.getLogger(__name__) +VALID_PLATFORMS = ["glue", "athena"] +DEFAULT_PLATFORM = "glue" -class GlueSourceConfig(AwsSourceConfig): + +class GlueSourceConfig(AwsSourceConfig, PlatformSourceConfigBase): extract_owners: Optional[bool] = True extract_transforms: Optional[bool] = True @@ -71,6 +75,29 @@ def check_direction(cls, v: str) -> str: ) return v.lower() + @validator("underlying_platform") + def underlying_platform_validator(cls, v: str) -> str: + if not v or v in VALID_PLATFORMS: + return v + else: + raise ConfigurationError( + f"'underlying_platform' can only take following values: {VALID_PLATFORMS}" + ) + + @validator("platform") + def platform_validator(cls, v: str) -> str: + if not v or v in VALID_PLATFORMS: + return v + else: + raise ConfigurationError( + f"'platform' can only take following values: {VALID_PLATFORMS}" + ) + + @validator("platform_instance") + def platform_instance_validator(cls, v): + # TODO: is there any restriction on platform_instance values? + return v + @dataclass class GlueSourceReport(SourceReport): @@ -96,7 +123,6 @@ def __init__(self, config: GlueSourceConfig, ctx: PipelineContext): self.glue_client = config.glue_client self.s3_client = config.s3_client self.extract_transforms = config.extract_transforms - self.underlying_platform = config.underlying_platform self.env = config.env @classmethod @@ -104,10 +130,18 @@ def create(cls, config_dict, ctx): config = GlueSourceConfig.parse_obj(config_dict) return cls(config, ctx) - def get_underlying_platform(self): - if self.underlying_platform in ["athena"]: - return self.underlying_platform - return "glue" + @property + def platform(self) -> str: + """ + This deprecates "underlying_platform" field in favour of the standard "platform" one, which has + more priority when both are defined. + :return: platform, otherwise underlying_platform, otherwise "glue" + """ + return ( + self.source_config.platform + or self.source_config.underlying_platform + or DEFAULT_PLATFORM + ) def get_all_jobs(self): """ @@ -231,7 +265,12 @@ def process_dataflow_node( full_table_name = f"{node_args['database']}.{node_args['table_name']}" # we know that the table will already be covered when ingesting Glue tables - node_urn = f"urn:li:dataset:(urn:li:dataPlatform:{self.get_underlying_platform()},{full_table_name},{self.env})" + node_urn = mce_builder.make_dataset_urn_with_platform_instance( + platform=self.platform, + name=full_table_name, + env=self.env, + platform_instance=self.source_config.platform_instance, + ) # if data object is S3 bucket elif node_args.get("connection_type") == "s3": @@ -558,7 +597,7 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: for job in self.get_all_jobs(): flow_urn = mce_builder.make_data_flow_urn( - self.get_underlying_platform(), job["Name"], self.env + self.platform, job["Name"], self.env ) flow_wu = self.get_dataflow_wu(flow_urn, job) @@ -670,13 +709,18 @@ def get_schema_metadata(glue_source: GlueSource) -> SchemaMetadata: schemaName=table_name, version=0, fields=fields, - platform=f"urn:li:dataPlatform:{self.get_underlying_platform()}", + platform=f"urn:li:dataPlatform:{self.platform}", hash="", platformSchema=MySqlDDL(tableSchema=""), ) dataset_snapshot = DatasetSnapshot( - urn=f"urn:li:dataset:(urn:li:dataPlatform:{self.get_underlying_platform()},{table_name},{self.env})", + urn=mce_builder.make_dataset_urn_with_platform_instance( + platform=self.platform, + name=table_name, + env=self.env, + platform_instance=self.source_config.platform_instance, + ), aspects=[], ) diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json new file mode 100644 index 00000000000000..753d97401dd656 --- /dev/null +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -0,0 +1,1082 @@ +[ + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:owner", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "CrawlerSchemaDeserializerVersion": "1.0", + "CrawlerSchemaSerializerVersion": "1.0", + "UPDATED_BY_CRAWLER": "flights-crawler", + "averageRecordSize": "55", + "avro.schema.literal": "{\"type\":\"record\",\"name\":\"flights_avro_subset\",\"namespace\":\"default\",\"fields\":[{\"name\":\"yr\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"flightdate\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"uniquecarrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"airlineid\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"flightnum\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"origin\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"dest\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"depdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrierdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"weatherdelay\",\"type\":[\"null\",\"int\"],\"default\":null}]}", + "classification": "avro", + "compressionType": "none", + "objectCount": "30", + "recordCount": "169222196", + "sizeKey": "9503351413", + "typeOfData": "file", + "Location": "s3://crawler-public-us-west-2/flight/avro/", + "InputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat", + "OutputFormat": "org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat", + "Compressed": "False", + "NumberOfBuckets": "-1", + "SerdeInfo": "{'SerializationLibrary': 'org.apache.hadoop.hive.serde2.avro.AvroSerDe', 'Parameters': {'avro.schema.literal': '{\"type\":\"record\",\"name\":\"flights_avro_subset\",\"namespace\":\"default\",\"fields\":[{\"name\":\"yr\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"flightdate\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"uniquecarrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"airlineid\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrier\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"flightnum\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"origin\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"dest\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"depdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"carrierdelay\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"weatherdelay\",\"type\":[\"null\",\"int\"],\"default\":null}]}', 'serialization.format': '1'}}", + "BucketColumns": "[]", + "SortColumns": "[]", + "StoredAsSubDirectories": "False" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "flights-database.avro", + "platform": "urn:li:dataPlatform:glue", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].yr", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].flightdate", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].uniquecarrier", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].airlineid", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].carrier", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].flightnum", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].origin", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].year", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_jsons_markers,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:owner", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "CrawlerSchemaDeserializerVersion": "1.0", + "CrawlerSchemaSerializerVersion": "1.0", + "UPDATED_BY_CRAWLER": "test-jsons", + "averageRecordSize": "273", + "classification": "json", + "compressionType": "none", + "objectCount": "1", + "recordCount": "1", + "sizeKey": "273", + "typeOfData": "file", + "Location": "s3://test-glue-jsons/markers/", + "InputFormat": "org.apache.hadoop.mapred.TextInputFormat", + "OutputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", + "Compressed": "False", + "NumberOfBuckets": "-1", + "SerdeInfo": "{'SerializationLibrary': 'org.openx.data.jsonserde.JsonSerDe', 'Parameters': {'paths': 'markers'}}", + "BucketColumns": "[]", + "SortColumns": "[]", + "StoredAsSubDirectories": "False" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-database.test_jsons_markers", + "platform": "urn:li:dataPlatform:glue", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "array,location:array>>", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array,location:array>>\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=string].name", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=array].[type=double].position", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "array", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=struct].markers.[type=array].[type=double].location", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": null + } + } + }, + "nativeDataType": "array", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"array\"}" + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:owner", + "type": "DATAOWNER", + "source": null + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + } + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "CrawlerSchemaDeserializerVersion": "1.0", + "CrawlerSchemaSerializerVersion": "1.0", + "UPDATED_BY_CRAWLER": "test", + "averageRecordSize": "19", + "classification": "parquet", + "compressionType": "none", + "objectCount": "60", + "recordCount": "167497743", + "sizeKey": "4463574900", + "typeOfData": "file", + "Location": "s3://crawler-public-us-west-2/flight/parquet/", + "InputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + "OutputFormat": "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat", + "Compressed": "False", + "NumberOfBuckets": "-1", + "SerdeInfo": "{'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe', 'Parameters': {'serialization.format': '1'}}", + "BucketColumns": "[]", + "SortColumns": "[]", + "StoredAsSubDirectories": "False" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "test-database.test_parquet", + "platform": "urn:li:dataPlatform:glue", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=int].yr", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].quarter", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].month", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=int].dayofmonth", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "int", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"int\", \"_nullable\": true}" + }, + { + "fieldPath": "[version=2.0].[type=string].year", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(glue,test-job-1,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": { + "role": "arn:aws:iam::123412341234:role/service-role/AWSGlueServiceRole-glue-crawler", + "created": "2021-06-10 16:51:25.690000", + "modified": "2021-06-10 16:55:35.307000", + "command": "s3://aws-glue-assets-123412341234-us-west-2/scripts/job-1.py" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1", + "description": "The first test job", + "project": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataFlowSnapshot": { + "urn": "urn:li:dataFlow:(glue,test-job-2,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataFlowInfo": { + "customProperties": { + "role": "arn:aws:iam::123412341234:role/service-role/AWSGlueServiceRole-glue-crawler", + "created": "2021-06-10 16:58:32.469000", + "modified": "2021-06-10 16:58:32.469000", + "command": "s3://aws-glue-assets-123412341234-us-west-2/scripts/job-2.py" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2", + "description": "The second test job", + "project": null + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "f": "lambda row : ()", + "transformation_ctx": "\"Transform0\"", + "transformType": "Filter", + "nodeId": "Transform0_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:Filter-Transform0_job1", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform1_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", + "transformation_ctx": "\"Transform1\"", + "transformType": "ApplyMapping", + "nodeId": "Transform1_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform1_job1", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", + "transformation_ctx": "\"Transform2\"", + "transformType": "ApplyMapping", + "nodeId": "Transform2_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform2_job1", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Join-Transform3_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "keys2": "[\"(right) flightdate\"]", + "transformation_ctx": "\"Transform3\"", + "keys1": "[\"yr\"]", + "transformType": "Join", + "nodeId": "Transform3_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:Join-Transform3_job1", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\"), (\"flightnum\", \"string\", \"flightnum\", \"string\"), (\"origin\", \"string\", \"origin\", \"string\"), (\"dest\", \"string\", \"dest\", \"string\"), (\"depdelay\", \"int\", \"depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"weatherdelay\", \"int\"), (\"year\", \"string\", \"year\", \"string\")]", + "transformation_ctx": "\"Transform4\"", + "transformType": "ApplyMapping", + "nodeId": "Transform4_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform4_job1", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform5_job1)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"(right) yr\", \"int\"), (\"flightdate\", \"string\", \"(right) flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"(right) uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"(right) airlineid\", \"int\"), (\"carrier\", \"string\", \"(right) carrier\", \"string\"), (\"flightnum\", \"string\", \"(right) flightnum\", \"string\"), (\"origin\", \"string\", \"(right) origin\", \"string\"), (\"dest\", \"string\", \"(right) dest\", \"string\"), (\"depdelay\", \"int\", \"(right) depdelay\", \"int\"), (\"carrierdelay\", \"int\", \"(right) carrierdelay\", \"int\"), (\"weatherdelay\", \"int\", \"(right) weatherdelay\", \"int\"), (\"year\", \"string\", \"(right) year\", \"string\")]", + "transformation_ctx": "\"Transform5\"", + "transformType": "ApplyMapping", + "nodeId": "Transform5_job1" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-1/graph", + "name": "test-job-1:ApplyMapping-Transform5_job1", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "connection_type": "s3", + "format": "json", + "connection_options": "{'path': 's3://test-glue-jsons/', 'partitionKeys': []}", + "transformation_ctx": "DataSink1" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SplitFields-Transform0_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "paths": "[\"yr\", \"quarter\", \"month\", \"dayofmonth\", \"dayofweek\", \"flightdate\", \"uniquecarrier\"]", + "name2": "\"Transform0Output1\"", + "name1": "\"Transform0Output0\"", + "transformation_ctx": "\"Transform0\"", + "transformType": "SplitFields", + "nodeId": "Transform0_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:SplitFields-Transform0_job2", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "mappings": "[(\"yr\", \"int\", \"yr\", \"int\"), (\"quarter\", \"int\", \"quarter\", \"int\"), (\"month\", \"int\", \"month\", \"int\"), (\"dayofmonth\", \"int\", \"dayofmonth\", \"int\"), (\"dayofweek\", \"int\", \"dayofweek\", \"int\"), (\"flightdate\", \"string\", \"flightdate\", \"string\"), (\"uniquecarrier\", \"string\", \"uniquecarrier\", \"string\"), (\"airlineid\", \"int\", \"airlineid\", \"int\"), (\"carrier\", \"string\", \"carrier\", \"string\")]", + "transformation_ctx": "\"Transform1\"", + "transformType": "ApplyMapping", + "nodeId": "Transform1_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:ApplyMapping-Transform1_job2", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)" + ], + "outputDatasets": [], + "inputDatajobs": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "missing_values_column": "\"dayofmonth\"", + "transformation_ctx": "\"Transform2\"", + "transformType": "FillMissingValues", + "nodeId": "Transform2_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:FillMissingValues-Transform2_job2", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DataJobSnapshot": { + "urn": "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),SelectFields-Transform3_job2)", + "aspects": [ + { + "com.linkedin.pegasus2avro.datajob.DataJobInfo": { + "customProperties": { + "paths": "[]", + "transformation_ctx": "\"Transform3\"", + "transformType": "SelectFields", + "nodeId": "Transform3_job2" + }, + "externalUrl": "https://us-west-2.console.aws.amazon.com/gluestudio/home?region=us-west-2#/editor/job/test-job-2/graph", + "name": "test-job-2:SelectFields-Transform3_job2", + "description": null, + "type": { + "string": "GLUE" + }, + "flowUrn": null, + "status": null + } + }, + { + "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { + "inputDatasets": [], + "outputDatasets": [ + "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)" + ], + "inputDatajobs": [ + "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)" + ] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + }, + { + "auditHeader": null, + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:s3,test-glue-jsons,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "connection_type": "s3", + "format": "json", + "connection_options": "{'path': 's3://test-glue-jsons/', 'partitionKeys': []}", + "transformation_ctx": "DataSink0" + }, + "externalUrl": null, + "description": null, + "uri": null, + "tags": [] + } + } + ] + } + }, + "proposedDelta": null, + "systemMetadata": null + } +] \ No newline at end of file diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index e6439c6ef684f8..8a2847355a2a90 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -1,10 +1,11 @@ import json -from typing import Dict, Tuple, Type +from typing import Dict, Optional, Tuple, Type import pytest from botocore.stub import Stubber from freezegun import freeze_time +from datahub.configuration.common import ConfigurationError from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.extractor.schema_util import avro_schema_to_mce_fields from datahub.ingestion.source.aws.glue import GlueSource, GlueSourceConfig @@ -32,10 +33,14 @@ FROZEN_TIME = "2020-04-14 07:00:00" -def glue_source() -> GlueSource: +def glue_source(platform_instance: Optional[str] = None) -> GlueSource: return GlueSource( ctx=PipelineContext(run_id="glue-source-test"), - config=GlueSourceConfig(aws_region="us-west-2", extract_transforms=True), + config=GlueSourceConfig( + aws_region="us-west-2", + extract_transforms=True, + platform_instance=platform_instance, + ), ) @@ -61,10 +66,23 @@ def test_column_type(hive_column_type: str, expected_type: Type) -> None: assert type(actual_schema_field_type.type) == expected_type +@pytest.mark.parametrize( + "platform_instance, mce_file, mce_golden_file", + [ + (None, "glue_mces.json", "glue_mces_golden.json"), + ( + "some_instance_name", + "glue_mces_platform_instance.json", + "glue_mces_platform_instance_golden.json", + ), + ], +) @freeze_time(FROZEN_TIME) -def test_glue_ingest(tmp_path, pytestconfig): +def test_glue_ingest( + tmp_path, pytestconfig, platform_instance, mce_file, mce_golden_file +): - glue_source_instance = glue_source() + glue_source_instance = glue_source(platform_instance=platform_instance) with Stubber(glue_source_instance.glue_client) as glue_stubber: @@ -95,7 +113,7 @@ def test_glue_ingest(tmp_path, pytestconfig): s3_stubber.add_response( "get_object", - get_object_response_1, + get_object_response_1(), { "Bucket": "aws-glue-assets-123412341234-us-west-2", "Key": "scripts/job-1.py", @@ -103,7 +121,7 @@ def test_glue_ingest(tmp_path, pytestconfig): ) s3_stubber.add_response( "get_object", - get_object_response_2, + get_object_response_2(), { "Bucket": "aws-glue-assets-123412341234-us-west-2", "Key": "scripts/job-2.py", @@ -114,15 +132,18 @@ def test_glue_ingest(tmp_path, pytestconfig): wu.metadata.to_obj() for wu in glue_source_instance.get_workunits() ] - with open(str(tmp_path / "glue_mces.json"), "w") as f: + glue_stubber.assert_no_pending_responses() + s3_stubber.assert_no_pending_responses() + + with open(str(tmp_path / mce_file), "w") as f: json.dump(mce_objects, f, indent=2) # Verify the output. test_resources_dir = pytestconfig.rootpath / "tests/unit/glue" mce_helpers.check_golden_file( pytestconfig, - output_path=tmp_path / "glue_mces.json", - golden_path=test_resources_dir / "glue_mces_golden.json", + output_path=tmp_path / mce_file, + golden_path=test_resources_dir / mce_golden_file, ) @@ -131,17 +152,35 @@ def test_underlying_platform_takes_precendence(): ctx=PipelineContext(run_id="glue-source-test"), config=GlueSourceConfig(aws_region="us-west-2", underlying_platform="athena"), ) - assert source.get_underlying_platform() == "athena" + assert source.platform == "athena" -def test_underlying_platform_cannot_be_other_than_athena(): +def test_platform_takes_precendence_over_underlying_platform(): source = GlueSource( ctx=PipelineContext(run_id="glue-source-test"), config=GlueSourceConfig( - aws_region="us-west-2", underlying_platform="data-warehouse" + aws_region="us-west-2", platform="athena", underlying_platform="glue" ), ) - assert source.get_underlying_platform() == "glue" + assert source.platform == "athena" + + +def test_underlying_platform_must_be_valid(): + with pytest.raises(ConfigurationError): + GlueSource( + ctx=PipelineContext(run_id="glue-source-test"), + config=GlueSourceConfig( + aws_region="us-west-2", underlying_platform="data-warehouse" + ), + ) + + +def test_platform_must_be_valid(): + with pytest.raises(ConfigurationError): + GlueSource( + ctx=PipelineContext(run_id="glue-source-test"), + config=GlueSourceConfig(aws_region="us-west-2", platform="data-warehouse"), + ) def test_without_underlying_platform(): @@ -149,4 +188,4 @@ def test_without_underlying_platform(): ctx=PipelineContext(run_id="glue-source-test"), config=GlueSourceConfig(aws_region="us-west-2"), ) - assert source.get_underlying_platform() == "glue" + assert source.platform == "glue" diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/test_glue_source_stubs.py index 77db442d5b1345..ef0e7e97cd37ab 100644 --- a/metadata-ingestion/tests/unit/test_glue_source_stubs.py +++ b/metadata-ingestion/tests/unit/test_glue_source_stubs.py @@ -759,5 +759,9 @@ def mock_get_object_response(raw_body: str) -> Dict[str, Any]: return {"Body": raw_stream} -get_object_response_1 = mock_get_object_response(get_object_body_1) -get_object_response_2 = mock_get_object_response(get_object_body_2) +def get_object_response_1(): + return mock_get_object_response(get_object_body_1) + + +def get_object_response_2(): + return mock_get_object_response(get_object_body_2) From 0ecf83dc9337bb517bcc47af055f98951ea117e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Feb 2022 17:42:57 +0100 Subject: [PATCH 02/12] refactor(ingestion): fixing some typings from PR review --- .../src/datahub/ingestion/source/aws/glue.py | 8 ++------ metadata-ingestion/tests/unit/test_glue_source.py | 10 ++++++++-- .../tests/unit/test_glue_source_stubs.py | 4 ++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index b14aec05690ff6..962c4c30a20de8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -46,8 +46,9 @@ logger = logging.getLogger(__name__) -VALID_PLATFORMS = ["glue", "athena"] + DEFAULT_PLATFORM = "glue" +VALID_PLATFORMS = [DEFAULT_PLATFORM, "athena"] class GlueSourceConfig(AwsSourceConfig, PlatformSourceConfigBase): @@ -93,11 +94,6 @@ def platform_validator(cls, v: str) -> str: f"'platform' can only take following values: {VALID_PLATFORMS}" ) - @validator("platform_instance") - def platform_instance_validator(cls, v): - # TODO: is there any restriction on platform_instance values? - return v - @dataclass class GlueSourceReport(SourceReport): diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index 8a2847355a2a90..012ee8604bd908 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -1,4 +1,5 @@ import json +from pathlib import Path from typing import Dict, Optional, Tuple, Type import pytest @@ -17,6 +18,7 @@ ) from datahub.utilities.hive_schema_to_avro import get_avro_schema_for_hive_column from tests.test_helpers import mce_helpers +from tests.test_helpers.type_helpers import PytestConfig from tests.unit.test_glue_source_stubs import ( get_databases_response, get_dataflow_graph_response_1, @@ -79,8 +81,12 @@ def test_column_type(hive_column_type: str, expected_type: Type) -> None: ) @freeze_time(FROZEN_TIME) def test_glue_ingest( - tmp_path, pytestconfig, platform_instance, mce_file, mce_golden_file -): + tmp_path: Path, + pytestconfig: PytestConfig, + platform_instance: str, + mce_file: str, + mce_golden_file: str, +) -> None: glue_source_instance = glue_source(platform_instance=platform_instance) diff --git a/metadata-ingestion/tests/unit/test_glue_source_stubs.py b/metadata-ingestion/tests/unit/test_glue_source_stubs.py index ef0e7e97cd37ab..a1ddc2edceaecb 100644 --- a/metadata-ingestion/tests/unit/test_glue_source_stubs.py +++ b/metadata-ingestion/tests/unit/test_glue_source_stubs.py @@ -759,9 +759,9 @@ def mock_get_object_response(raw_body: str) -> Dict[str, Any]: return {"Body": raw_stream} -def get_object_response_1(): +def get_object_response_1() -> Dict[str, Any]: return mock_get_object_response(get_object_body_1) -def get_object_response_2(): +def get_object_response_2() -> Dict[str, Any]: return mock_get_object_response(get_object_body_2) From 6db9022fb0ee085dfb7b931b9183a020e93d76c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Feb 2022 19:38:49 +0100 Subject: [PATCH 03/12] fix(ingestion): container custom property instance now points to the platform instance for the glue connector --- .../src/datahub/ingestion/source/aws/glue.py | 2 +- .../tests/unit/glue/glue_mces_golden.json | 22 +++++++++---------- .../glue_mces_platform_instance_golden.json | 22 +++++++++---------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index 5a8354fe0986a3..e312e65c8b4b9b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -573,7 +573,7 @@ def gen_database_key(self, database: str) -> DatabaseKey: return DatabaseKey( database=database, platform=self.platform, - instance=self.env, + instance=self.source_config.platform_instance or "", ) def gen_database_containers(self, database: str) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index eb8a9269ad90ec..c5d1c99e2dc3d6 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", "contentType": "application/json" }, "systemMetadata": null @@ -15,7 +15,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", @@ -28,7 +28,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -276,7 +276,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba\"}", + "value": "{\"container\": \"urn:li:container:9d886605e90cf7f36a21437113dcb82a\"}", "contentType": "application/json" }, "systemMetadata": null @@ -284,12 +284,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", "contentType": "application/json" }, "systemMetadata": null @@ -297,7 +297,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", @@ -310,7 +310,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -495,7 +495,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", + "value": "{\"container\": \"urn:li:container:db56f9cccad12a624dd082c839b2445b\"}", "contentType": "application/json" }, "systemMetadata": null @@ -686,7 +686,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", + "value": "{\"container\": \"urn:li:container:db56f9cccad12a624dd082c839b2445b\"}", "contentType": "application/json" }, "systemMetadata": null diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 23380b44a6f1dc..21d8764dc11eec 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", "contentType": "application/json" }, "systemMetadata": null @@ -15,7 +15,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", @@ -28,7 +28,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", + "entityUrn": "urn:li:container:7d53111f2c71396ea6f6d26c84770665", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -276,7 +276,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba\"}", + "value": "{\"container\": \"urn:li:container:7d53111f2c71396ea6f6d26c84770665\"}", "contentType": "application/json" }, "systemMetadata": null @@ -284,12 +284,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"some_instance_name\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", "contentType": "application/json" }, "systemMetadata": null @@ -297,7 +297,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", @@ -310,7 +310,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", + "entityUrn": "urn:li:container:9fb26491b2c92dde9e80791dbecca9ca", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -495,7 +495,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", + "value": "{\"container\": \"urn:li:container:9fb26491b2c92dde9e80791dbecca9ca\"}", "contentType": "application/json" }, "systemMetadata": null @@ -686,7 +686,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", + "value": "{\"container\": \"urn:li:container:9fb26491b2c92dde9e80791dbecca9ca\"}", "contentType": "application/json" }, "systemMetadata": null From 31da10f18b0d05ab58ceb04a3f2921fd0b6f4271 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Feb 2022 21:32:31 +0100 Subject: [PATCH 04/12] fix(ingestion): container custom properties filters null values --- metadata-ingestion/src/datahub/emitter/mcp_builder.py | 6 +++++- metadata-ingestion/src/datahub/ingestion/source/aws/glue.py | 2 +- metadata-ingestion/tests/unit/glue/glue_mces_golden.json | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index d70dd50afacbc5..7413d8b5d1b19c 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -90,7 +90,11 @@ def gen_containers( # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), aspectName="containerProperties", aspect=ContainerProperties( - name=name, customProperties=dataclasses.asdict(container_key) + name=name, + customProperties=dataclasses.asdict( + container_key, + dict_factory=lambda x: {k: v for (k, v) in x if v is not None}, + ), ), ) wu = MetadataWorkUnit(id=f"container-info-{name}-{container_urn}", mcp=mcp) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index e312e65c8b4b9b..6ef5ff5f925016 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -573,7 +573,7 @@ def gen_database_key(self, database: str) -> DatabaseKey: return DatabaseKey( database=database, platform=self.platform, - instance=self.source_config.platform_instance or "", + instance=self.source_config.platform_instance, ) def gen_database_containers(self, database: str) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index c5d1c99e2dc3d6..f12b4bbff49977 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -7,7 +7,7 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", "contentType": "application/json" }, "systemMetadata": null @@ -289,7 +289,7 @@ "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", "contentType": "application/json" }, "systemMetadata": null From 26ef3ba1762fc491b015f5ec1c9aca13e0c44c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Feb 2022 21:53:13 +0100 Subject: [PATCH 05/12] fix(ingestion): container produces dataPlatformInstance mcp only if instance is given --- .../src/datahub/emitter/mcp_builder.py | 34 +++++++++++-------- .../tests/unit/glue/glue_mces_golden.json | 26 -------------- .../glue_mces_platform_instance_golden.json | 4 +-- 3 files changed, 21 insertions(+), 43 deletions(-) diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index 7413d8b5d1b19c..17189060e844e4 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -3,7 +3,10 @@ import json from typing import Any, Iterable, List, Optional, TypeVar, Union -from datahub.emitter.mce_builder import make_container_urn, make_data_platform_urn +from datahub.emitter.mce_builder import ( + make_container_urn, + make_dataplatform_instance_urn, +) from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.metadata.com.linkedin.pegasus2avro.common import DataPlatformInstance @@ -100,20 +103,21 @@ def gen_containers( wu = MetadataWorkUnit(id=f"container-info-{name}-{container_urn}", mcp=mcp) yield wu - mcp = MetadataChangeProposalWrapper( - entityType="container", - changeType=ChangeTypeClass.UPSERT, - entityUrn=f"{container_urn}", - # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), - aspectName="dataPlatformInstance", - aspect=DataPlatformInstance( - platform=f"{make_data_platform_urn(container_key.platform)}" - ), - ) - wu = MetadataWorkUnit( - id=f"container-platforminstance-{name}-{container_urn}", mcp=mcp - ) - yield wu + if container_key.instance: + mcp = MetadataChangeProposalWrapper( + entityType="container", + changeType=ChangeTypeClass.UPSERT, + entityUrn=f"{container_urn}", + # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), + aspectName="dataPlatformInstance", + aspect=DataPlatformInstance( + platform=f"{make_dataplatform_instance_urn(container_key.platform, container_key.instance)}" + ), + ) + wu = MetadataWorkUnit( + id=f"container-platforminstance-{name}-{container_urn}", mcp=mcp + ) + yield wu # Set subtype subtype_mcp = MetadataChangeProposalWrapper( diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index f12b4bbff49977..d7eb01bd8f8eef 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -12,19 +12,6 @@ }, "systemMetadata": null }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", - "contentType": "application/json" - }, - "systemMetadata": null - }, { "auditHeader": null, "entityType": "container", @@ -294,19 +281,6 @@ }, "systemMetadata": null }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", - "contentType": "application/json" - }, - "systemMetadata": null - }, { "auditHeader": null, "entityType": "container", diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 21d8764dc11eec..4e0c0b1ed2142d 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -20,7 +20,7 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", + "value": "{\"platform\": \"urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)\"}", "contentType": "application/json" }, "systemMetadata": null @@ -302,7 +302,7 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", + "value": "{\"platform\": \"urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)\"}", "contentType": "application/json" }, "systemMetadata": null From 59050d47c7ef84786964da13b0349ba0e006774e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Feb 2022 22:06:31 +0100 Subject: [PATCH 06/12] fix(ingestion): fixes instances = platform_instance for DatabaseKey and SchemaKey for athena and sql common --- .../src/datahub/ingestion/source/sql/athena.py | 4 +++- .../src/datahub/ingestion/source/sql/sql_common.py | 4 ++-- metadata-ingestion/tests/unit/test_mcp_builder.py | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py index 27fce4cb9d33ec..243abec109d0a8 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/athena.py @@ -105,7 +105,9 @@ def gen_database_containers( def gen_schema_key(self, db_name: str, schema: str) -> DatabaseKey: return DatabaseKey( - platform=self.platform, instance=self.config.env, database=schema + platform=self.platform, + instance=self.config.platform_instance, + database=schema, ) def gen_schema_containers( diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py index ff5aee21e4e085..8305c8d442773b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/sql_common.py @@ -555,14 +555,14 @@ def gen_schema_key(self, db_name: str, schema: str) -> PlatformKey: database=db_name, schema=schema, platform=self.platform, - instance=self.config.env, + instance=self.config.platform_instance, ) def gen_database_key(self, database: str) -> PlatformKey: return DatabaseKey( database=database, platform=self.platform, - instance=self.config.env, + instance=self.config.platform_instance, ) def gen_database_containers(self, database: str) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/tests/unit/test_mcp_builder.py b/metadata-ingestion/tests/unit/test_mcp_builder.py index 0825de2340403a..f12229b6d50dd8 100644 --- a/metadata-ingestion/tests/unit/test_mcp_builder.py +++ b/metadata-ingestion/tests/unit/test_mcp_builder.py @@ -4,11 +4,11 @@ def test_guid_generator(): key = builder.SchemaKey( - database="test", schema="Test", platform="mysql", instance="PROD" + database="test", schema="Test", platform="mysql", instance="some_mysql" ) guid = key.guid() - assert guid == "f5268c71373b9100d50c1299861cfb3f" + assert guid == "cc05baee5809b42df29c8e550c12b153" def test_guid_generator_with_empty_instance(): @@ -22,7 +22,7 @@ def test_guid_generator_with_empty_instance(): def test_guid_generators(): key = builder.SchemaKey( - database="test", schema="Test", platform="mysql", instance="PROD" + database="test", schema="Test", platform="mysql", instance="some_mysql" ) guid_datahub = datahub_guid(key.__dict__) From 98d9956b1a82060860f0137f21f2f02e99eec2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Feb 2022 23:19:16 +0100 Subject: [PATCH 07/12] fix(ingestion): integration tests --- .../integration/hive/hive_mces_golden.json | 96 +- .../integration/mysql/mysql_mces_golden.json | 159 +- .../sql_server/mssql_mces_golden.json | 1935 +++++++---------- .../trino/trino_hive_mces_golden.json | 90 +- .../integration/trino/trino_mces_golden.json | 62 +- 5 files changed, 924 insertions(+), 1418 deletions(-) diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json index f6bac4fb1094dd..ff0ade07dd45b5 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", + "entityUrn": "urn:li:container:c2b113857642266679292bc25d7b803b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"db1\"}, \"name\": \"db1\"}", + "value": "{\"customProperties\": {\"platform\": \"hive\", \"database\": \"db1\"}, \"name\": \"db1\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,26 +21,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "hive-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", + "entityUrn": "urn:li:container:c2b113857642266679292bc25d7b803b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -59,31 +40,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", + "entityUrn": "urn:li:container:d075957c1b090705368c0fa651ee494f", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"db1\", \"schema\": \"db1\"}, \"name\": \"db1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "hive-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "value": "{\"customProperties\": {\"platform\": \"hive\", \"database\": \"db1\", \"schema\": \"db1\"}, \"name\": \"db1\"}", "contentType": "application/json" }, "systemMetadata": { @@ -97,7 +59,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", + "entityUrn": "urn:li:container:d075957c1b090705368c0fa651ee494f", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -116,12 +78,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", + "entityUrn": "urn:li:container:d075957c1b090705368c0fa651ee494f", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:ded36d15fcfbbb939830549697122661\"}", + "value": "{\"container\": \"urn:li:container:c2b113857642266679292bc25d7b803b\"}", "contentType": "application/json" }, "systemMetadata": { @@ -140,7 +102,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", + "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", "contentType": "application/json" }, "systemMetadata": { @@ -167,7 +129,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Fri Feb 04 16:55:17 UTC 2022", + "CreateTime:": "Thu Feb 17 21:49:35 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -177,7 +139,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1643993717", + "Table Parameters: transient_lastDdlTime": "1645134575", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -278,7 +240,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", + "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", "contentType": "application/json" }, "systemMetadata": { @@ -305,7 +267,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Fri Feb 04 16:55:17 UTC 2022", + "CreateTime:": "Thu Feb 17 21:49:35 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -315,7 +277,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1643993720", + "Table Parameters: transient_lastDdlTime": "1645134580", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -454,7 +416,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", + "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", "contentType": "application/json" }, "systemMetadata": { @@ -481,7 +443,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Fri Feb 04 16:55:21 UTC 2022", + "CreateTime:": "Thu Feb 17 21:49:41 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -491,7 +453,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1643993721", + "Table Parameters: transient_lastDdlTime": "1645134581", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -595,7 +557,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", + "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", "contentType": "application/json" }, "systemMetadata": { @@ -622,7 +584,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Fri Feb 04 16:55:20 UTC 2022", + "CreateTime:": "Thu Feb 17 21:49:41 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -632,7 +594,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1643993720", + "Table Parameters: transient_lastDdlTime": "1645134581", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -801,7 +763,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", + "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", "contentType": "application/json" }, "systemMetadata": { @@ -828,7 +790,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Fri Feb 04 16:55:13 UTC 2022", + "CreateTime:": "Thu Feb 17 21:49:31 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -837,7 +799,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1643993715", + "Table Parameters: transient_lastDdlTime": "1645134573", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -938,7 +900,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", + "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", "contentType": "application/json" }, "systemMetadata": { @@ -965,7 +927,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Fri Feb 04 16:55:17 UTC 2022", + "CreateTime:": "Thu Feb 17 21:49:35 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -975,7 +937,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1643993717", + "Table Parameters: transient_lastDdlTime": "1645134575", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1112,7 +1074,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", + "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1139,7 +1101,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Fri Feb 04 16:55:20 UTC 2022", + "CreateTime:": "Thu Feb 17 21:49:41 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -1149,7 +1111,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1643993720", + "Table Parameters: transient_lastDdlTime": "1645134581", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json index 106a623c371e97..e5b0dd362a35b9 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityUrn": "urn:li:container:475c4a082c95a7428e6d919a07d06030", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,26 +21,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityUrn": "urn:li:container:475c4a082c95a7428e6d919a07d06030", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -59,7 +40,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityUrn": "urn:li:container:475c4a082c95a7428e6d919a07d06030", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", @@ -78,31 +59,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityUrn": "urn:li:container:bd134461dbd95a1cffdd7120da4a752b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"datacharmer\"}, \"name\": \"datacharmer\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"datacharmer\"}, \"name\": \"datacharmer\"}", "contentType": "application/json" }, "systemMetadata": { @@ -116,7 +78,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityUrn": "urn:li:container:bd134461dbd95a1cffdd7120da4a752b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -135,12 +97,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityUrn": "urn:li:container:bd134461dbd95a1cffdd7120da4a752b", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", "contentType": "application/json" }, "systemMetadata": { @@ -159,7 +121,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", + "value": "{\"container\": \"urn:li:container:bd134461dbd95a1cffdd7120da4a752b\"}", "contentType": "application/json" }, "systemMetadata": { @@ -334,7 +296,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", + "value": "{\"container\": \"urn:li:container:bd134461dbd95a1cffdd7120da4a752b\"}", "contentType": "application/json" }, "systemMetadata": { @@ -470,31 +432,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityUrn": "urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", "contentType": "application/json" }, "systemMetadata": { @@ -508,7 +451,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityUrn": "urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -527,12 +470,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityUrn": "urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", "contentType": "application/json" }, "systemMetadata": { @@ -551,7 +494,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", + "value": "{\"container\": \"urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a\"}", "contentType": "application/json" }, "systemMetadata": { @@ -762,7 +705,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", + "value": "{\"container\": \"urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a\"}", "contentType": "application/json" }, "systemMetadata": { @@ -982,7 +925,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", + "value": "{\"container\": \"urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1149,31 +1092,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityUrn": "urn:li:container:001cbec12033cd997c8100f154945d66", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"northwind\"}, \"name\": \"northwind\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"northwind\"}, \"name\": \"northwind\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1187,7 +1111,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityUrn": "urn:li:container:001cbec12033cd997c8100f154945d66", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -1206,12 +1130,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityUrn": "urn:li:container:001cbec12033cd997c8100f154945d66", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1230,7 +1154,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", + "value": "{\"container\": \"urn:li:container:001cbec12033cd997c8100f154945d66\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1405,7 +1329,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", + "value": "{\"container\": \"urn:li:container:001cbec12033cd997c8100f154945d66\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1535,31 +1459,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityUrn": "urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"test_cases\"}, \"name\": \"test_cases\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1586847600000, - "runId": "mysql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"test_cases\"}, \"name\": \"test_cases\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1573,7 +1478,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityUrn": "urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -1592,12 +1497,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityUrn": "urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", + "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1616,7 +1521,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:989c003cbe689094c2b5c340a67f62be\"}", + "value": "{\"container\": \"urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779\"}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json b/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json index 560ceb191aa9fb..51ce33dcce6586 100644 --- a/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json +++ b/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json @@ -1,822 +1,613 @@ [ -{ + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "entityUrn": "urn:li:container:26847d592c367190435b875dc666d0a7", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\"}, \"name\": \"demodata\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\"}, \"name\": \"demodata\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "entityUrn": "urn:li:container:26847d592c367190435b875dc666d0a7", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Database\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Database\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "entityUrn": "urn:li:container:d37f2b34d17df8ef094dc6787d1f3f77", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_accessadmin\"}, \"name\": \"db_accessadmin\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_accessadmin\"}, \"name\": \"db_accessadmin\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "entityUrn": "urn:li:container:d37f2b34d17df8ef094dc6787d1f3f77", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "entityUrn": "urn:li:container:d37f2b34d17df8ef094dc6787d1f3f77", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "entityUrn": "urn:li:container:5142c7fde67c7f992b2069da55671508", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_backupoperator\"}, \"name\": \"db_backupoperator\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_backupoperator\"}, \"name\": \"db_backupoperator\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "entityUrn": "urn:li:container:5142c7fde67c7f992b2069da55671508", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "entityUrn": "urn:li:container:5142c7fde67c7f992b2069da55671508", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "entityUrn": "urn:li:container:2b91f7fd2c4f70917335f6cdc8510f05", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datareader\"}, \"name\": \"db_datareader\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_datareader\"}, \"name\": \"db_datareader\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "entityUrn": "urn:li:container:2b91f7fd2c4f70917335f6cdc8510f05", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "entityUrn": "urn:li:container:2b91f7fd2c4f70917335f6cdc8510f05", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "entityUrn": "urn:li:container:f232ce677dcefc31090f006110d7e008", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datawriter\"}, \"name\": \"db_datawriter\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_datawriter\"}, \"name\": \"db_datawriter\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "entityUrn": "urn:li:container:f232ce677dcefc31090f006110d7e008", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "entityUrn": "urn:li:container:f232ce677dcefc31090f006110d7e008", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "entityUrn": "urn:li:container:4e395dfc46b722195beb69772431b1ca", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_ddladmin\"}, \"name\": \"db_ddladmin\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_ddladmin\"}, \"name\": \"db_ddladmin\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "entityUrn": "urn:li:container:4e395dfc46b722195beb69772431b1ca", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "entityUrn": "urn:li:container:4e395dfc46b722195beb69772431b1ca", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "entityUrn": "urn:li:container:c6a77b98350a18f3a3ed0447b2a1f089", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatareader\"}, \"name\": \"db_denydatareader\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_denydatareader\"}, \"name\": \"db_denydatareader\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "entityUrn": "urn:li:container:c6a77b98350a18f3a3ed0447b2a1f089", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "entityUrn": "urn:li:container:c6a77b98350a18f3a3ed0447b2a1f089", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "entityUrn": "urn:li:container:dd33b8c89ddef76cb18d1b007c76ae42", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatawriter\"}, \"name\": \"db_denydatawriter\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_denydatawriter\"}, \"name\": \"db_denydatawriter\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "entityUrn": "urn:li:container:dd33b8c89ddef76cb18d1b007c76ae42", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "entityUrn": "urn:li:container:dd33b8c89ddef76cb18d1b007c76ae42", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "entityUrn": "urn:li:container:5398f92d79e31613262f5bec78d1658c", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_owner\"}, \"name\": \"db_owner\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_owner\"}, \"name\": \"db_owner\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "entityUrn": "urn:li:container:5398f92d79e31613262f5bec78d1658c", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "entityUrn": "urn:li:container:5398f92d79e31613262f5bec78d1658c", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "entityUrn": "urn:li:container:e4f53f1fa011fa06d75558a214e3e125", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_securityadmin\"}, \"name\": \"db_securityadmin\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_securityadmin\"}, \"name\": \"db_securityadmin\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "entityUrn": "urn:li:container:e4f53f1fa011fa06d75558a214e3e125", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "entityUrn": "urn:li:container:e4f53f1fa011fa06d75558a214e3e125", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "entityUrn": "urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"dbo\"}, \"name\": \"dbo\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"dbo\"}, \"name\": \"dbo\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "entityUrn": "urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "entityUrn": "urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", @@ -824,182 +615,163 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d41a036a2e6cfa44b834edf7683199ec\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.dbo.products,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.dbo.Products", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "ID", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.dbo.Products", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "ID", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ProductName", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "NVARCHAR()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "ProductName", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "NVARCHAR()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "entityUrn": "urn:li:container:3b9bef99b5b889a835f874573b1d17f5", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"Foo\"}, \"name\": \"Foo\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"Foo\"}, \"name\": \"Foo\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "entityUrn": "urn:li:container:3b9bef99b5b889a835f874573b1d17f5", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "entityUrn": "urn:li:container:3b9bef99b5b889a835f874573b1d17f5", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", @@ -1007,106 +779,106 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:3b9bef99b5b889a835f874573b1d17f5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.items,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.Items", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "ID", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.Foo.Items", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "ID", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "ItemName", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "NVARCHAR()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "ItemName", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "NVARCHAR()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", @@ -1114,140 +886,140 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:3b9bef99b5b889a835f874573b1d17f5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.Persons", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "ID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.Foo.Persons", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "ID", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "LastName", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "FirstName", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - }, - { - "fieldPath": "Age", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + "fieldPath": "LastName", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "FirstName", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "VARCHAR(length=255, collation='SQL_Latin1_General_CP1_CI_AS')", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + }, + { + "fieldPath": "Age", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null } - ] - } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": null + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", @@ -1255,342 +1027,285 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:3b9bef99b5b889a835f874573b1d17f5\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "proposedSnapshot": { - "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", - "aspects": [ + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "DemoData.Foo.SalesReason", + "platform": "urn:li:dataPlatform:mssql", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown", + "impersonator": null + }, + "deleted": null, + "dataset": null, + "cluster": null, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.MySqlDDL": { + "tableSchema": "" + } + }, + "fields": [ { - "com.linkedin.pegasus2avro.common.Status": { - "removed": false + "fieldPath": "TempID", + "jsonPath": null, + "nullable": false, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} } + }, + "nativeDataType": "INTEGER()", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": true, + "jsonProps": null }, { - "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "DemoData.Foo.SalesReason", - "platform": "urn:li:dataPlatform:mssql", - "version": 0, - "created": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "lastModified": { - "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null - }, - "deleted": null, - "dataset": null, - "cluster": null, - "hash": "", - "platformSchema": { - "com.linkedin.pegasus2avro.schema.MySqlDDL": { - "tableSchema": "" - } - }, - "fields": [ - { - "fieldPath": "TempID", - "jsonPath": null, - "nullable": false, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.NumberType": {} - } - }, - "nativeDataType": "INTEGER()", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": true, - "jsonProps": null - }, - { - "fieldPath": "Name", - "jsonPath": null, - "nullable": true, - "description": null, - "type": { - "type": { - "com.linkedin.pegasus2avro.schema.StringType": {} - } - }, - "nativeDataType": "NVARCHAR(length=50)", - "recursive": false, - "globalTags": null, - "glossaryTerms": null, - "isPartOfKey": false, - "jsonProps": null - } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": [ - { - "name": "FK_TempSales_SalesReason", - "foreignFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" - ], - "sourceFields": [ - "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)" - ], - "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" - } - ] + "fieldPath": "Name", + "jsonPath": null, + "nullable": true, + "description": null, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} } + }, + "nativeDataType": "NVARCHAR(length=50)", + "recursive": false, + "globalTags": null, + "glossaryTerms": null, + "isPartOfKey": false, + "jsonProps": null + } + ], + "primaryKeys": null, + "foreignKeysSpecs": null, + "foreignKeys": [ + { + "name": "FK_TempSales_SalesReason", + "foreignFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD),ID)" + ], + "sourceFields": [ + "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.salesreason,PROD),TempID)" + ], + "foreignDataset": "urn:li:dataset:(urn:li:dataPlatform:mssql,demodata.foo.persons,PROD)" } - ] - } + ] + } + } + ] + } }, "proposedDelta": null, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "entityUrn": "urn:li:container:ad64df64f7a8291209ca694c061f8ca0", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"guest\"}, \"name\": \"guest\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"guest\"}, \"name\": \"guest\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "entityUrn": "urn:li:container:ad64df64f7a8291209ca694c061f8ca0", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "entityUrn": "urn:li:container:ad64df64f7a8291209ca694c061f8ca0", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "entityUrn": "urn:li:container:e87a3a3313b3e3f94b4a81c760b348e2", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"INFORMATION_SCHEMA\"}, \"name\": \"INFORMATION_SCHEMA\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"INFORMATION_SCHEMA\"}, \"name\": \"INFORMATION_SCHEMA\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "entityUrn": "urn:li:container:e87a3a3313b3e3f94b4a81c760b348e2", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "entityUrn": "urn:li:container:e87a3a3313b3e3f94b4a81c760b348e2", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "entityUrn": "urn:li:container:440377895af34d0a2044a05069c5d369", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"sys\"}, \"name\": \"sys\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null - } -}, -{ - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", - "contentType": "application/json" + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"sys\"}, \"name\": \"sys\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "entityUrn": "urn:li:container:440377895af34d0a2044a05069c5d369", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { - "value": "{\"typeNames\": [\"Schema\"]}", - "contentType": "application/json" + "value": "{\"typeNames\": [\"Schema\"]}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -}, -{ + }, + { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "entityUrn": "urn:li:container:440377895af34d0a2044a05069c5d369", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", - "contentType": "application/json" + "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "contentType": "application/json" }, "systemMetadata": { - "lastObserved": 1615443388097, - "runId": "mssql-test", - "registryName": null, - "registryVersion": null, - "properties": null + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null } -} + } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index aa8f2664a74543..7d32837ef3c088 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "entityUrn": "urn:li:container:9178e668559c5bd02628404522271670", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"hivedb\"}, \"name\": \"hivedb\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"hivedb\"}, \"name\": \"hivedb\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,26 +21,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-hive-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "entityUrn": "urn:li:container:9178e668559c5bd02628404522271670", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -59,31 +40,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "entityUrn": "urn:li:container:5954737a5eb17f0af92654a8bbbf918c", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"hivedb\", \"schema\": \"db1\"}, \"name\": \"db1\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-hive-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"hivedb\", \"schema\": \"db1\"}, \"name\": \"db1\"}", "contentType": "application/json" }, "systemMetadata": { @@ -97,7 +59,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "entityUrn": "urn:li:container:5954737a5eb17f0af92654a8bbbf918c", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -116,12 +78,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "entityUrn": "urn:li:container:5954737a5eb17f0af92654a8bbbf918c", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7\"}", + "value": "{\"container\": \"urn:li:container:9178e668559c5bd02628404522271670\"}", "contentType": "application/json" }, "systemMetadata": { @@ -140,7 +102,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -172,7 +134,7 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1643991015" + "transient_lastddltime": "1645133485" }, "externalUrl": null, "description": "This table has array of structs", @@ -303,7 +265,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -333,7 +295,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1643991018" + "transient_lastddltime": "1645133488" }, "externalUrl": null, "description": null, @@ -429,7 +391,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -459,7 +421,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1643991017" + "transient_lastddltime": "1645133488" }, "externalUrl": null, "description": null, @@ -620,7 +582,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -645,7 +607,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1643991008" + "transient_lastddltime": "1645133480" }, "externalUrl": null, "description": null, @@ -755,7 +717,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -785,7 +747,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1643991011" + "transient_lastddltime": "1645133482" }, "externalUrl": null, "description": null, @@ -914,7 +876,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -941,7 +903,7 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1643991017" + "transient_lastddltime": "1645133488" }, "externalUrl": null, "description": null, @@ -1070,7 +1032,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1100,7 +1062,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1643991011" + "transient_lastddltime": "1645133482" }, "externalUrl": null, "description": null, @@ -1193,7 +1155,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1223,7 +1185,7 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1643991018" + "transient_lastddltime": "1645133488" }, "externalUrl": null, "description": null, @@ -1420,7 +1382,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", + "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1445,7 +1407,7 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1643991017", + "transient_lastddltime": "1645133488", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, diff --git a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json index abcc99aa63df2b..b922cc0092491d 100644 --- a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:57aba13b10c1691508600999cd411c25", + "entityUrn": "urn:li:container:5c9faca60223215dd4a55bc23490f0ba", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"library_catalog\"}, \"name\": \"library_catalog\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"library_catalog\"}, \"name\": \"library_catalog\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,26 +21,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:57aba13b10c1691508600999cd411c25", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:57aba13b10c1691508600999cd411c25", + "entityUrn": "urn:li:container:5c9faca60223215dd4a55bc23490f0ba", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -59,31 +40,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "entityUrn": "urn:li:container:2aea0448929baa6c73cee3834074f196", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"library_catalog\", \"schema\": \"librarydb\"}, \"name\": \"librarydb\"}", - "contentType": "application/json" - }, - "systemMetadata": { - "lastObserved": 1632398400000, - "runId": "trino-test", - "registryName": null, - "registryVersion": null, - "properties": null - } - }, - { - "auditHeader": null, - "entityType": "container", - "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", - "entityKeyAspect": null, - "changeType": "UPSERT", - "aspectName": "dataPlatformInstance", - "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"library_catalog\", \"schema\": \"librarydb\"}, \"name\": \"librarydb\"}", "contentType": "application/json" }, "systemMetadata": { @@ -97,7 +59,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "entityUrn": "urn:li:container:2aea0448929baa6c73cee3834074f196", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -116,12 +78,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "entityUrn": "urn:li:container:2aea0448929baa6c73cee3834074f196", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:57aba13b10c1691508600999cd411c25\"}", + "value": "{\"container\": \"urn:li:container:5c9faca60223215dd4a55bc23490f0ba\"}", "contentType": "application/json" }, "systemMetadata": { @@ -140,7 +102,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", + "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", "contentType": "application/json" }, "systemMetadata": { @@ -317,7 +279,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", + "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", "contentType": "application/json" }, "systemMetadata": { @@ -458,7 +420,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", + "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", "contentType": "application/json" }, "systemMetadata": { @@ -565,7 +527,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", + "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", "contentType": "application/json" }, "systemMetadata": { From a641b377c85251091f511e48fa02a624ca4e7086 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Mar 2022 17:26:09 +0100 Subject: [PATCH 08/12] fixing tests --- .../src/datahub/emitter/mcp_builder.py | 29 +- .../src/datahub/ingestion/source/aws/glue.py | 8 + .../integration/hive/hive_mces_golden.json | 110 +++-- .../integration/mysql/mysql_mces_golden.json | 163 +++++-- .../sql_server/mssql_mces_golden.json | 439 +++++++++++++++--- .../trino/trino_hive_mces_golden.json | 108 +++-- .../integration/trino/trino_mces_golden.json | 62 ++- .../tests/unit/glue/glue_mces_golden.json | 104 ++++- .../glue_mces_platform_instance_golden.json | 82 +++- 9 files changed, 892 insertions(+), 213 deletions(-) diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py index 3ce1eadd95d96d..8a737f34c8a3d7 100644 --- a/metadata-ingestion/src/datahub/emitter/mcp_builder.py +++ b/metadata-ingestion/src/datahub/emitter/mcp_builder.py @@ -151,21 +151,20 @@ def gen_containers( wu = MetadataWorkUnit(id=f"container-info-{name}-{container_urn}", mcp=mcp) yield wu - if container_key.instance: - mcp = MetadataChangeProposalWrapper( - entityType="container", - changeType=ChangeTypeClass.UPSERT, - entityUrn=f"{container_urn}", - # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), - aspectName="dataPlatformInstance", - aspect=DataPlatformInstance( - platform=f"{make_dataplatform_instance_urn(container_key.platform, container_key.instance)}" - ), - ) - wu = MetadataWorkUnit( - id=f"container-platforminstance-{name}-{container_urn}", mcp=mcp - ) - yield wu + mcp = MetadataChangeProposalWrapper( + entityType="container", + changeType=ChangeTypeClass.UPSERT, + entityUrn=f"{container_urn}", + # entityKeyAspect=ContainerKeyClass(guid=schema_container_key.guid()), + aspectName="dataPlatformInstance", + aspect=DataPlatformInstance( + platform=f"{make_data_platform_urn(container_key.platform)}" + ), + ) + wu = MetadataWorkUnit( + id=f"container-platforminstance-{name}-{container_urn}", mcp=mcp + ) + yield wu # Set subtype subtype_mcp = MetadataChangeProposalWrapper( diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index 4423624fb1d99c..950b25304d6495 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -43,6 +43,7 @@ DataJobInfoClass, DataJobInputOutputClass, DataJobSnapshotClass, + DataPlatformInstanceClass, DatasetLineageTypeClass, DatasetPropertiesClass, MetadataChangeEventClass, @@ -788,6 +789,12 @@ def get_schema_metadata(glue_source: GlueSource) -> SchemaMetadata: platformSchema=MySqlDDL(tableSchema=""), ) + def get_data_platform_instance() -> DataPlatformInstanceClass: + return DataPlatformInstanceClass( + platform=f"urn:li:dataPlatform:{self.platform}", + instance=self.source_config.platform_instance, + ) + dataset_snapshot = DatasetSnapshot( urn=make_dataset_urn_with_platform_instance( platform=self.platform, @@ -807,6 +814,7 @@ def get_schema_metadata(glue_source: GlueSource) -> SchemaMetadata: dataset_snapshot.aspects.append(get_dataset_properties()) dataset_snapshot.aspects.append(get_schema_metadata(self)) + dataset_snapshot.aspects.append(get_data_platform_instance()) metadata_record = MetadataChangeEvent(proposedSnapshot=dataset_snapshot) return metadata_record diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json index ff0ade07dd45b5..e5fdcd5201261b 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:c2b113857642266679292bc25d7b803b", + "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"database\": \"db1\"}, \"name\": \"db1\"}", + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"db1\"}, \"name\": \"db1\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,7 +21,26 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:c2b113857642266679292bc25d7b803b", + "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ded36d15fcfbbb939830549697122661", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -40,12 +59,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d075957c1b090705368c0fa651ee494f", + "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"hive\", \"database\": \"db1\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "value": "{\"customProperties\": {\"platform\": \"hive\", \"instance\": \"PROD\", \"database\": \"db1\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "hive-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:hive\"}", "contentType": "application/json" }, "systemMetadata": { @@ -59,7 +97,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d075957c1b090705368c0fa651ee494f", + "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -78,12 +116,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d075957c1b090705368c0fa651ee494f", + "entityUrn": "urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:c2b113857642266679292bc25d7b803b\"}", + "value": "{\"container\": \"urn:li:container:ded36d15fcfbbb939830549697122661\"}", "contentType": "application/json" }, "systemMetadata": { @@ -102,7 +140,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", + "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", "contentType": "application/json" }, "systemMetadata": { @@ -129,7 +167,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Feb 17 21:49:35 UTC 2022", + "CreateTime:": "Thu Mar 17 15:35:49 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -139,7 +177,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1645134575", + "Table Parameters: transient_lastDdlTime": "1647531349", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -150,6 +188,8 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -240,7 +280,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", + "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", "contentType": "application/json" }, "systemMetadata": { @@ -267,7 +307,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Feb 17 21:49:35 UTC 2022", + "CreateTime:": "Thu Mar 17 15:35:49 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -277,7 +317,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1645134580", + "Table Parameters: transient_lastDdlTime": "1647531354", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -288,6 +328,8 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -416,7 +458,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", + "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", "contentType": "application/json" }, "systemMetadata": { @@ -443,7 +485,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Feb 17 21:49:41 UTC 2022", + "CreateTime:": "Thu Mar 17 15:35:55 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -453,7 +495,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1645134581", + "Table Parameters: transient_lastDdlTime": "1647531355", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -464,6 +506,8 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -557,7 +601,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", + "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", "contentType": "application/json" }, "systemMetadata": { @@ -584,7 +628,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Feb 17 21:49:41 UTC 2022", + "CreateTime:": "Thu Mar 17 15:35:55 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -594,7 +638,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1645134581", + "Table Parameters: transient_lastDdlTime": "1647531355", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -605,6 +649,8 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -763,7 +809,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", + "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", "contentType": "application/json" }, "systemMetadata": { @@ -790,7 +836,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Feb 17 21:49:31 UTC 2022", + "CreateTime:": "Thu Mar 17 15:35:45 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -799,7 +845,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1645134573", + "Table Parameters: transient_lastDdlTime": "1647531347", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -810,6 +856,8 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -900,7 +948,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", + "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", "contentType": "application/json" }, "systemMetadata": { @@ -927,7 +975,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Feb 17 21:49:35 UTC 2022", + "CreateTime:": "Thu Mar 17 15:35:49 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -937,7 +985,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1645134575", + "Table Parameters: transient_lastDdlTime": "1647531349", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -948,6 +996,8 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1074,7 +1124,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:d075957c1b090705368c0fa651ee494f\"}", + "value": "{\"container\": \"urn:li:container:d4ee8f0f53fee8e83d4188d0497bfe37\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1101,7 +1151,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Feb 17 21:49:41 UTC 2022", + "CreateTime:": "Thu Mar 17 15:35:55 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -1111,7 +1161,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1645134581", + "Table Parameters: transient_lastDdlTime": "1647531355", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1122,6 +1172,8 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json index e5b0dd362a35b9..bc189879ed83dc 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:475c4a082c95a7428e6d919a07d06030", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,7 +21,26 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:475c4a082c95a7428e6d919a07d06030", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -40,7 +59,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:475c4a082c95a7428e6d919a07d06030", + "entityUrn": "urn:li:container:17751259af32dd0385cad799df608c40", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "domains", @@ -59,12 +78,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bd134461dbd95a1cffdd7120da4a752b", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"datacharmer\"}, \"name\": \"datacharmer\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"datacharmer\"}, \"name\": \"datacharmer\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -78,7 +116,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bd134461dbd95a1cffdd7120da4a752b", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -97,12 +135,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:bd134461dbd95a1cffdd7120da4a752b", + "entityUrn": "urn:li:container:ba408413d97771e6470c16f9869f2e0d", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", "contentType": "application/json" }, "systemMetadata": { @@ -121,7 +159,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:bd134461dbd95a1cffdd7120da4a752b\"}", + "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", "contentType": "application/json" }, "systemMetadata": { @@ -296,7 +334,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:bd134461dbd95a1cffdd7120da4a752b\"}", + "value": "{\"container\": \"urn:li:container:ba408413d97771e6470c16f9869f2e0d\"}", "contentType": "application/json" }, "systemMetadata": { @@ -432,12 +470,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"metagalaxy\"}, \"name\": \"metagalaxy\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -451,7 +508,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -470,12 +527,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a", + "entityUrn": "urn:li:container:593ea3998729fdae4bdfb42206561a3a", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", "contentType": "application/json" }, "systemMetadata": { @@ -494,7 +551,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a\"}", + "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", "contentType": "application/json" }, "systemMetadata": { @@ -705,7 +762,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a\"}", + "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", "contentType": "application/json" }, "systemMetadata": { @@ -731,6 +788,8 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": "This is a table comment", "uri": null, "tags": [] @@ -925,7 +984,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:2ee86fd5d1601cf2b56da4ec234bd89a\"}", + "value": "{\"container\": \"urn:li:container:593ea3998729fdae4bdfb42206561a3a\"}", "contentType": "application/json" }, "systemMetadata": { @@ -954,6 +1013,8 @@ "is_view": "True" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1092,12 +1153,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:001cbec12033cd997c8100f154945d66", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"northwind\"}, \"name\": \"northwind\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"northwind\"}, \"name\": \"northwind\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1111,7 +1191,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:001cbec12033cd997c8100f154945d66", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -1130,12 +1210,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:001cbec12033cd997c8100f154945d66", + "entityUrn": "urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1154,7 +1234,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:001cbec12033cd997c8100f154945d66\"}", + "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1329,7 +1409,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:001cbec12033cd997c8100f154945d66\"}", + "value": "{\"container\": \"urn:li:container:36bfb6eae3f7972efbcb56dedecdfba6\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1459,12 +1539,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"database\": \"metagalaxy\", \"schema\": \"test_cases\"}, \"name\": \"test_cases\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"metagalaxy\", \"schema\": \"test_cases\"}, \"name\": \"test_cases\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "mysql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mysql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1478,7 +1577,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -1497,12 +1596,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779", + "entityUrn": "urn:li:container:989c003cbe689094c2b5c340a67f62be", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:475c4a082c95a7428e6d919a07d06030\"}", + "value": "{\"container\": \"urn:li:container:17751259af32dd0385cad799df608c40\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1521,7 +1620,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:88e1c87399d45eaf6a5af8b0afd1d779\"}", + "value": "{\"container\": \"urn:li:container:989c003cbe689094c2b5c340a67f62be\"}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json b/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json index 2fae99645fd2c1..7e4a118788098e 100644 --- a/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json +++ b/metadata-ingestion/tests/integration/sql_server/mssql_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:26847d592c367190435b875dc666d0a7", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\"}, \"name\": \"demodata\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\"}, \"name\": \"demodata\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,7 +21,26 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:26847d592c367190435b875dc666d0a7", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -40,12 +59,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d37f2b34d17df8ef094dc6787d1f3f77", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_accessadmin\"}, \"name\": \"db_accessadmin\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_accessadmin\"}, \"name\": \"db_accessadmin\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -59,7 +97,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d37f2b34d17df8ef094dc6787d1f3f77", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -78,12 +116,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:d37f2b34d17df8ef094dc6787d1f3f77", + "entityUrn": "urn:li:container:f1b4c0e379c4b2e2e09a8ecd6c1b6dec", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -97,12 +135,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5142c7fde67c7f992b2069da55671508", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_backupoperator\"}, \"name\": \"db_backupoperator\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_backupoperator\"}, \"name\": \"db_backupoperator\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -116,7 +173,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5142c7fde67c7f992b2069da55671508", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -135,12 +192,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5142c7fde67c7f992b2069da55671508", + "entityUrn": "urn:li:container:bad84e08ecf49aee863df68243d8b9d0", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -154,12 +211,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2b91f7fd2c4f70917335f6cdc8510f05", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_datareader\"}, \"name\": \"db_datareader\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datareader\"}, \"name\": \"db_datareader\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -173,7 +249,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2b91f7fd2c4f70917335f6cdc8510f05", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -192,12 +268,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2b91f7fd2c4f70917335f6cdc8510f05", + "entityUrn": "urn:li:container:e48d82445eeacfbe13b431f0bb1826ee", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -211,12 +287,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:f232ce677dcefc31090f006110d7e008", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_datawriter\"}, \"name\": \"db_datawriter\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_datawriter\"}, \"name\": \"db_datawriter\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -230,7 +325,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:f232ce677dcefc31090f006110d7e008", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -249,12 +344,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:f232ce677dcefc31090f006110d7e008", + "entityUrn": "urn:li:container:884bfecd9e414990a494681293413e8e", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -268,12 +363,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:4e395dfc46b722195beb69772431b1ca", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_ddladmin\"}, \"name\": \"db_ddladmin\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_ddladmin\"}, \"name\": \"db_ddladmin\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -287,7 +401,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:4e395dfc46b722195beb69772431b1ca", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -306,12 +420,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:4e395dfc46b722195beb69772431b1ca", + "entityUrn": "urn:li:container:142ca5fc51b7f44e5e6a424bf1043590", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -325,12 +439,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:c6a77b98350a18f3a3ed0447b2a1f089", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_denydatareader\"}, \"name\": \"db_denydatareader\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatareader\"}, \"name\": \"db_denydatareader\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -344,7 +477,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:c6a77b98350a18f3a3ed0447b2a1f089", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -363,12 +496,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:c6a77b98350a18f3a3ed0447b2a1f089", + "entityUrn": "urn:li:container:1b9d125d390447de36719bfb8dd1f782", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -382,12 +515,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:dd33b8c89ddef76cb18d1b007c76ae42", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_denydatawriter\"}, \"name\": \"db_denydatawriter\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_denydatawriter\"}, \"name\": \"db_denydatawriter\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -401,7 +553,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:dd33b8c89ddef76cb18d1b007c76ae42", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -420,12 +572,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:dd33b8c89ddef76cb18d1b007c76ae42", + "entityUrn": "urn:li:container:fcd4c8da3739150766f91e7f6c2a3a30", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -439,12 +591,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5398f92d79e31613262f5bec78d1658c", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_owner\"}, \"name\": \"db_owner\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_owner\"}, \"name\": \"db_owner\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -458,7 +629,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5398f92d79e31613262f5bec78d1658c", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -477,12 +648,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5398f92d79e31613262f5bec78d1658c", + "entityUrn": "urn:li:container:2029cab615b3cd82cb87b153957d2e92", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -496,12 +667,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e4f53f1fa011fa06d75558a214e3e125", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"db_securityadmin\"}, \"name\": \"db_securityadmin\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"db_securityadmin\"}, \"name\": \"db_securityadmin\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -515,7 +705,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e4f53f1fa011fa06d75558a214e3e125", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -534,12 +724,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e4f53f1fa011fa06d75558a214e3e125", + "entityUrn": "urn:li:container:556e25ccec98892284f017f870ef7809", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -553,12 +743,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"dbo\"}, \"name\": \"dbo\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"dbo\"}, \"name\": \"dbo\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -572,7 +781,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -591,12 +800,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8", + "entityUrn": "urn:li:container:d41a036a2e6cfa44b834edf7683199ec", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -615,7 +824,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:a8b183484cf1e4dcacc671542d2a1cc8\"}", + "value": "{\"container\": \"urn:li:container:d41a036a2e6cfa44b834edf7683199ec\"}", "contentType": "application/json" }, "systemMetadata": { @@ -717,12 +926,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:3b9bef99b5b889a835f874573b1d17f5", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"Foo\"}, \"name\": \"Foo\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"Foo\"}, \"name\": \"Foo\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -736,7 +964,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:3b9bef99b5b889a835f874573b1d17f5", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -755,12 +983,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:3b9bef99b5b889a835f874573b1d17f5", + "entityUrn": "urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -779,7 +1007,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:3b9bef99b5b889a835f874573b1d17f5\"}", + "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", "contentType": "application/json" }, "systemMetadata": { @@ -886,7 +1114,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:3b9bef99b5b889a835f874573b1d17f5\"}", + "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1027,7 +1255,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:3b9bef99b5b889a835f874573b1d17f5\"}", + "value": "{\"container\": \"urn:li:container:6e5c6d608d0a2dcc4eb03591382e5671\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1140,12 +1368,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ad64df64f7a8291209ca694c061f8ca0", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"guest\"}, \"name\": \"guest\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"guest\"}, \"name\": \"guest\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1159,7 +1406,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ad64df64f7a8291209ca694c061f8ca0", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -1178,12 +1425,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:ad64df64f7a8291209ca694c061f8ca0", + "entityUrn": "urn:li:container:a6bea84fba7b05fb5d12630c8e6306ac", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1197,12 +1444,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e87a3a3313b3e3f94b4a81c760b348e2", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"INFORMATION_SCHEMA\"}, \"name\": \"INFORMATION_SCHEMA\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"INFORMATION_SCHEMA\"}, \"name\": \"INFORMATION_SCHEMA\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1216,7 +1482,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e87a3a3313b3e3f94b4a81c760b348e2", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -1235,12 +1501,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:e87a3a3313b3e3f94b4a81c760b348e2", + "entityUrn": "urn:li:container:9f37bb7baa7ded19cc023e9f644a8cf8", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1254,12 +1520,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:440377895af34d0a2044a05069c5d369", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mssql\", \"database\": \"demodata\", \"schema\": \"sys\"}, \"name\": \"sys\"}", + "value": "{\"customProperties\": {\"platform\": \"mssql\", \"instance\": \"PROD\", \"database\": \"demodata\", \"schema\": \"sys\"}, \"name\": \"sys\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "mssql-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:mssql\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1273,7 +1558,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:440377895af34d0a2044a05069c5d369", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -1292,12 +1577,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:440377895af34d0a2044a05069c5d369", + "entityUrn": "urn:li:container:3f157d8292fb473142f19e2250af537f", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:26847d592c367190435b875dc666d0a7\"}", + "value": "{\"container\": \"urn:li:container:b7062d1c0c650d9de0f7a9a5de00b1b5\"}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index 7d32837ef3c088..96dfe98a7aed54 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9178e668559c5bd02628404522271670", + "entityUrn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"hivedb\"}, \"name\": \"hivedb\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"hivedb\"}, \"name\": \"hivedb\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,7 +21,26 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9178e668559c5bd02628404522271670", + "entityUrn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -40,12 +59,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5954737a5eb17f0af92654a8bbbf918c", + "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"hivedb\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"hivedb\", \"schema\": \"db1\"}, \"name\": \"db1\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-hive-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", "contentType": "application/json" }, "systemMetadata": { @@ -59,7 +97,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5954737a5eb17f0af92654a8bbbf918c", + "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -78,12 +116,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5954737a5eb17f0af92654a8bbbf918c", + "entityUrn": "urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:9178e668559c5bd02628404522271670\"}", + "value": "{\"container\": \"urn:li:container:c7a81f6ed9a7cdd0c74436ac2dc4d1f7\"}", "contentType": "application/json" }, "systemMetadata": { @@ -102,7 +140,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -134,9 +172,11 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1645133485" + "transient_lastddltime": "1647532365" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": "This table has array of structs", "uri": null, "tags": [] @@ -265,7 +305,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -295,9 +335,11 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1645133488" + "transient_lastddltime": "1647532369" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -391,7 +433,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -421,9 +463,11 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1645133488" + "transient_lastddltime": "1647532369" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -582,7 +626,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -607,9 +651,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1645133480" + "transient_lastddltime": "1647532356" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -717,7 +763,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -747,9 +793,11 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1645133482" + "transient_lastddltime": "1647532360" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -876,7 +924,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -903,9 +951,11 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1645133488" + "transient_lastddltime": "1647532368" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1032,7 +1082,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1062,9 +1112,11 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1645133482" + "transient_lastddltime": "1647532359" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1155,7 +1207,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1185,9 +1237,11 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1645133488" + "transient_lastddltime": "1647532369" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1382,7 +1436,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5954737a5eb17f0af92654a8bbbf918c\"}", + "value": "{\"container\": \"urn:li:container:304fd7ad57dc0ab32fb2cb778cbccd84\"}", "contentType": "application/json" }, "systemMetadata": { @@ -1407,11 +1461,13 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1645133488", + "transient_lastddltime": "1647532368", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] diff --git a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json index b922cc0092491d..abcc99aa63df2b 100644 --- a/metadata-ingestion/tests/integration/trino/trino_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5c9faca60223215dd4a55bc23490f0ba", + "entityUrn": "urn:li:container:57aba13b10c1691508600999cd411c25", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"library_catalog\"}, \"name\": \"library_catalog\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"library_catalog\"}, \"name\": \"library_catalog\"}", "contentType": "application/json" }, "systemMetadata": { @@ -21,7 +21,26 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:5c9faca60223215dd4a55bc23490f0ba", + "entityUrn": "urn:li:container:57aba13b10c1691508600999cd411c25", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:57aba13b10c1691508600999cd411c25", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -40,12 +59,31 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2aea0448929baa6c73cee3834074f196", + "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"trino\", \"database\": \"library_catalog\", \"schema\": \"librarydb\"}, \"name\": \"librarydb\"}", + "value": "{\"customProperties\": {\"platform\": \"trino\", \"instance\": \"PROD\", \"database\": \"library_catalog\", \"schema\": \"librarydb\"}, \"name\": \"librarydb\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1632398400000, + "runId": "trino-test", + "registryName": null, + "registryVersion": null, + "properties": null + } + }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:trino\"}", "contentType": "application/json" }, "systemMetadata": { @@ -59,7 +97,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2aea0448929baa6c73cee3834074f196", + "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -78,12 +116,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:2aea0448929baa6c73cee3834074f196", + "entityUrn": "urn:li:container:8568c0971ce28c183fde0dff7f88e617", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:5c9faca60223215dd4a55bc23490f0ba\"}", + "value": "{\"container\": \"urn:li:container:57aba13b10c1691508600999cd411c25\"}", "contentType": "application/json" }, "systemMetadata": { @@ -102,7 +140,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", + "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", "contentType": "application/json" }, "systemMetadata": { @@ -279,7 +317,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", + "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", "contentType": "application/json" }, "systemMetadata": { @@ -420,7 +458,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", + "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", "contentType": "application/json" }, "systemMetadata": { @@ -527,7 +565,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:2aea0448929baa6c73cee3834074f196\"}", + "value": "{\"container\": \"urn:li:container:8568c0971ce28c183fde0dff7f88e617\"}", "contentType": "application/json" }, "systemMetadata": { diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index d7eb01bd8f8eef..2e03f70fb58cc4 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -12,6 +12,19 @@ }, "systemMetadata": null }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, { "auditHeader": null, "entityType": "container", @@ -77,6 +90,8 @@ "StoredAsSubDirectories": "False" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -248,6 +263,12 @@ "foreignKeysSpecs": null, "foreignKeys": null } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue", + "instance": null + } } ] } @@ -281,6 +302,19 @@ }, "systemMetadata": null }, + { + "auditHeader": null, + "entityType": "container", + "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", + "entityKeyAspect": null, + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", + "contentType": "application/json" + }, + "systemMetadata": null + }, { "auditHeader": null, "entityType": "container", @@ -345,6 +379,8 @@ "StoredAsSubDirectories": "False" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -454,6 +490,12 @@ "foreignKeysSpecs": null, "foreignKeys": null } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue", + "instance": null + } } ] } @@ -525,6 +567,8 @@ "StoredAsSubDirectories": "False" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -645,6 +689,12 @@ "foreignKeysSpecs": null, "foreignKeys": null } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue", + "instance": null + } } ] } @@ -747,7 +797,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -786,7 +839,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -825,7 +881,10 @@ "urn:li:dataset:(urn:li:dataPlatform:glue,flights-database.avro,PROD)" ], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -865,7 +924,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -902,7 +964,10 @@ "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -939,7 +1004,10 @@ "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -968,6 +1036,8 @@ "transformation_ctx": "DataSink1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1011,7 +1081,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1050,7 +1123,10 @@ "urn:li:dataset:(urn:li:dataPlatform:glue,test-database.test_parquet,PROD)" ], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1089,7 +1165,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1130,7 +1209,10 @@ ], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1159,6 +1241,8 @@ "transformation_ctx": "DataSink0" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 4e0c0b1ed2142d..363399f434f50c 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -20,7 +20,7 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)\"}", + "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", "contentType": "application/json" }, "systemMetadata": null @@ -90,6 +90,8 @@ "StoredAsSubDirectories": "False" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -261,6 +263,12 @@ "foreignKeysSpecs": null, "foreignKeys": null } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue", + "instance": "some_instance_name" + } } ] } @@ -302,7 +310,7 @@ "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { - "value": "{\"platform\": \"urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)\"}", + "value": "{\"platform\": \"urn:li:dataPlatform:glue\"}", "contentType": "application/json" }, "systemMetadata": null @@ -371,6 +379,8 @@ "StoredAsSubDirectories": "False" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -480,6 +490,12 @@ "foreignKeysSpecs": null, "foreignKeys": null } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue", + "instance": "some_instance_name" + } } ] } @@ -551,6 +567,8 @@ "StoredAsSubDirectories": "False" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -671,6 +689,12 @@ "foreignKeysSpecs": null, "foreignKeys": null } + }, + { + "com.linkedin.pegasus2avro.common.DataPlatformInstance": { + "platform": "urn:li:dataPlatform:glue", + "instance": "some_instance_name" + } } ] } @@ -773,7 +797,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform2_job1)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -812,7 +839,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),Filter-Transform0_job1)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -851,7 +881,10 @@ "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.flights-database.avro,PROD)" ], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -891,7 +924,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-1,PROD),ApplyMapping-Transform4_job1)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -928,7 +964,10 @@ "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -965,7 +1004,10 @@ "com.linkedin.pegasus2avro.datajob.DataJobInputOutput": { "inputDatasets": [], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -994,6 +1036,8 @@ "transformation_ctx": "DataSink1" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1037,7 +1081,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1076,7 +1123,10 @@ "urn:li:dataset:(urn:li:dataPlatform:glue,some_instance_name.test-database.test_parquet,PROD)" ], "outputDatasets": [], - "inputDatajobs": [] + "inputDatajobs": [], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1115,7 +1165,10 @@ "outputDatasets": [], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),ApplyMapping-Transform1_job2)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1156,7 +1209,10 @@ ], "inputDatajobs": [ "urn:li:dataJob:(urn:li:dataFlow:(glue,test-job-2,PROD),FillMissingValues-Transform2_job2)" - ] + ], + "inputDatasetFields": null, + "outputDatasetFields": null, + "fineGrainedLineages": null } } ] @@ -1185,6 +1241,8 @@ "transformation_ctx": "DataSink0" }, "externalUrl": null, + "name": null, + "qualifiedName": null, "description": null, "uri": null, "tags": [] From 3179aedd273259806e6d0435b59fdfb5badb231b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Mar 2022 18:00:40 +0100 Subject: [PATCH 09/12] revert unnecessary changes on some mce files --- .../integration/hive/hive_mces_golden.json | 42 +++++++------------ .../integration/mysql/mysql_mces_golden.json | 4 -- .../trino/trino_hive_mces_golden.json | 36 ++++------------ 3 files changed, 23 insertions(+), 59 deletions(-) diff --git a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json index e5fdcd5201261b..f6bac4fb1094dd 100644 --- a/metadata-ingestion/tests/integration/hive/hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/hive/hive_mces_golden.json @@ -167,7 +167,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Mar 17 15:35:49 UTC 2022", + "CreateTime:": "Fri Feb 04 16:55:17 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/_test_table_underscore", @@ -177,7 +177,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1647531349", + "Table Parameters: transient_lastDdlTime": "1643993717", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -188,8 +188,6 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -307,7 +305,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Mar 17 15:35:49 UTC 2022", + "CreateTime:": "Fri Feb 04 16:55:17 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/array_struct_test", @@ -317,7 +315,7 @@ "Table Parameters: numRows": "1", "Table Parameters: rawDataSize": "32", "Table Parameters: totalSize": "33", - "Table Parameters: transient_lastDdlTime": "1647531354", + "Table Parameters: transient_lastDdlTime": "1643993720", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -328,8 +326,6 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -485,7 +481,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Mar 17 15:35:55 UTC 2022", + "CreateTime:": "Fri Feb 04 16:55:21 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/map_test", @@ -495,7 +491,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1647531355", + "Table Parameters: transient_lastDdlTime": "1643993721", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -506,8 +502,6 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -628,7 +622,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Mar 17 15:35:55 UTC 2022", + "CreateTime:": "Fri Feb 04 16:55:20 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/nested_struct_test", @@ -638,7 +632,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1647531355", + "Table Parameters: transient_lastDdlTime": "1643993720", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -649,8 +643,6 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -836,7 +828,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Mar 17 15:35:45 UTC 2022", + "CreateTime:": "Fri Feb 04 16:55:13 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/pokes", @@ -845,7 +837,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "5812", - "Table Parameters: transient_lastDdlTime": "1647531347", + "Table Parameters: transient_lastDdlTime": "1643993715", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -856,8 +848,6 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -975,7 +965,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Mar 17 15:35:49 UTC 2022", + "CreateTime:": "Fri Feb 04 16:55:17 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/struct_test", @@ -985,7 +975,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1647531349", + "Table Parameters: transient_lastDdlTime": "1643993717", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -996,8 +986,6 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1151,7 +1139,7 @@ "customProperties": { "Database:": "db1", "Owner:": "root", - "CreateTime:": "Thu Mar 17 15:35:55 UTC 2022", + "CreateTime:": "Fri Feb 04 16:55:20 UTC 2022", "LastAccessTime:": "UNKNOWN", "Retention:": "0", "Location:": "hdfs://namenode:8020/user/hive/warehouse/db1.db/union_test", @@ -1161,7 +1149,7 @@ "Table Parameters: numRows": "0", "Table Parameters: rawDataSize": "0", "Table Parameters: totalSize": "0", - "Table Parameters: transient_lastDdlTime": "1647531355", + "Table Parameters: transient_lastDdlTime": "1643993720", "SerDe Library:": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe", "InputFormat:": "org.apache.hadoop.mapred.TextInputFormat", "OutputFormat:": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat", @@ -1172,8 +1160,6 @@ "Storage Desc Params: serialization.format": "1" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json index bc189879ed83dc..106a623c371e97 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_golden.json @@ -788,8 +788,6 @@ "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": {}, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": "This is a table comment", "uri": null, "tags": [] @@ -1013,8 +1011,6 @@ "is_view": "True" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] diff --git a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json index 96dfe98a7aed54..aa8f2664a74543 100644 --- a/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json +++ b/metadata-ingestion/tests/integration/trino/trino_hive_mces_golden.json @@ -172,11 +172,9 @@ "numrows": "1", "rawdatasize": "32", "totalsize": "33", - "transient_lastddltime": "1647532365" + "transient_lastddltime": "1643991015" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": "This table has array of structs", "uri": null, "tags": [] @@ -335,11 +333,9 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1647532369" + "transient_lastddltime": "1643991018" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -463,11 +459,9 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1647532369" + "transient_lastddltime": "1643991017" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -651,11 +645,9 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1647532356" + "transient_lastddltime": "1643991008" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -793,11 +785,9 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1647532360" + "transient_lastddltime": "1643991011" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -951,11 +941,9 @@ "customProperties": { "numfiles": "0", "totalsize": "0", - "transient_lastddltime": "1647532368" + "transient_lastddltime": "1643991017" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1112,11 +1100,9 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1647532359" + "transient_lastddltime": "1643991011" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1237,11 +1223,9 @@ "numrows": "0", "rawdatasize": "0", "totalsize": "0", - "transient_lastddltime": "1647532369" + "transient_lastddltime": "1643991018" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] @@ -1461,13 +1445,11 @@ { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { - "transient_lastddltime": "1647532368", + "transient_lastddltime": "1643991017", "view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"", "is_view": "True" }, "externalUrl": null, - "name": null, - "qualifiedName": null, "description": null, "uri": null, "tags": [] From e9c05d5c424289603d7e494fc6ee874f891bc287 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Mar 2022 18:11:31 +0100 Subject: [PATCH 10/12] keep backward compatibility when no platform instance also for glue connector --- .../src/datahub/ingestion/source/aws/glue.py | 5 ++++- .../tests/unit/glue/glue_mces_golden.json | 22 +++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index 950b25304d6495..3e688d40f430b4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -573,7 +573,10 @@ def gen_database_key(self, database: str) -> DatabaseKey: return DatabaseKey( database=database, platform=self.platform, - instance=self.source_config.platform_instance, + instance=self.source_config.platform_instance + # keeps backward compatibility when platform instance is missed + if self.source_config.platform_instance is not None + else self.source_config.env, ) def gen_database_containers(self, database: str) -> Iterable[MetadataWorkUnit]: diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json index 2e03f70fb58cc4..96c040325b6cc9 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_golden.json @@ -2,12 +2,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"flights-database\"}, \"name\": \"flights-database\"}", "contentType": "application/json" }, "systemMetadata": null @@ -15,7 +15,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", @@ -28,7 +28,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:9d886605e90cf7f36a21437113dcb82a", + "entityUrn": "urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -284,7 +284,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:9d886605e90cf7f36a21437113dcb82a\"}", + "value": "{\"container\": \"urn:li:container:0b9f1f731ecf6743be6207fec3dc9cba\"}", "contentType": "application/json" }, "systemMetadata": null @@ -292,12 +292,12 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"glue\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", + "value": "{\"customProperties\": {\"platform\": \"glue\", \"instance\": \"PROD\", \"database\": \"test-database\"}, \"name\": \"test-database\"}", "contentType": "application/json" }, "systemMetadata": null @@ -305,7 +305,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "dataPlatformInstance", @@ -318,7 +318,7 @@ { "auditHeader": null, "entityType": "container", - "entityUrn": "urn:li:container:db56f9cccad12a624dd082c839b2445b", + "entityUrn": "urn:li:container:bdf4342ea6899d162eae685bfe9074a7", "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "subTypes", @@ -511,7 +511,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:db56f9cccad12a624dd082c839b2445b\"}", + "value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "contentType": "application/json" }, "systemMetadata": null @@ -710,7 +710,7 @@ "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:db56f9cccad12a624dd082c839b2445b\"}", + "value": "{\"container\": \"urn:li:container:bdf4342ea6899d162eae685bfe9074a7\"}", "contentType": "application/json" }, "systemMetadata": null From 37aaaea000399c15ae9a3dd32f24a7dc33597536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Mar 2022 18:24:27 +0100 Subject: [PATCH 11/12] fixing dataplatform aspect for glue datasets when platform instance is given --- .../src/datahub/ingestion/source/aws/glue.py | 10 ++++++++-- .../unit/glue/glue_mces_platform_instance_golden.json | 6 +++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py index 3e688d40f430b4..6024ab1c418dd6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py +++ b/metadata-ingestion/src/datahub/ingestion/source/aws/glue.py @@ -13,6 +13,8 @@ from datahub.configuration.source_common import PlatformSourceConfigBase from datahub.emitter import mce_builder from datahub.emitter.mce_builder import ( + make_data_platform_urn, + make_dataplatform_instance_urn, make_dataset_urn_with_platform_instance, make_domain_urn, ) @@ -794,8 +796,12 @@ def get_schema_metadata(glue_source: GlueSource) -> SchemaMetadata: def get_data_platform_instance() -> DataPlatformInstanceClass: return DataPlatformInstanceClass( - platform=f"urn:li:dataPlatform:{self.platform}", - instance=self.source_config.platform_instance, + platform=make_data_platform_urn(self.platform), + instance=make_dataplatform_instance_urn( + self.platform, self.source_config.platform_instance + ) + if self.source_config.platform_instance + else None, ) dataset_snapshot = DatasetSnapshot( diff --git a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json index 363399f434f50c..96ec4527c8237a 100644 --- a/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json +++ b/metadata-ingestion/tests/unit/glue/glue_mces_platform_instance_golden.json @@ -267,7 +267,7 @@ { "com.linkedin.pegasus2avro.common.DataPlatformInstance": { "platform": "urn:li:dataPlatform:glue", - "instance": "some_instance_name" + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" } } ] @@ -494,7 +494,7 @@ { "com.linkedin.pegasus2avro.common.DataPlatformInstance": { "platform": "urn:li:dataPlatform:glue", - "instance": "some_instance_name" + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" } } ] @@ -693,7 +693,7 @@ { "com.linkedin.pegasus2avro.common.DataPlatformInstance": { "platform": "urn:li:dataPlatform:glue", - "instance": "some_instance_name" + "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:glue,some_instance_name)" } } ] From 9a032b4ec8d2867060c96920f51e43a803ff0211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Go=CC=81mez?= Date: Thu, 17 Mar 2022 18:58:48 +0100 Subject: [PATCH 12/12] minor update --- metadata-ingestion/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 4628a40fba3595..23e6ee5f516cf6 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -104,7 +104,7 @@ def get_long_description(): trino = { "trino>=0.308", - "trino[sqlalchemy]>=0.308" + "trino[sqlalchemy]>=0.308", } microsoft_common = {"msal==1.16.0"}