From bfbbc7fcf8394dc6397a1ae5e95c5fdfe650e78d Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Mon, 28 Feb 2022 15:07:14 +0530 Subject: [PATCH 1/2] fix(ingestion): revert positional arg change --- metadata-ingestion/examples/library/dataset_add_column_tag.py | 1 + metadata-ingestion/examples/library/dataset_add_column_term.py | 1 + .../examples/library/dataset_add_documentation.py | 2 ++ metadata-ingestion/examples/library/dataset_add_owner.py | 1 + metadata-ingestion/examples/library/dataset_add_tag.py | 1 + metadata-ingestion/examples/library/dataset_add_term.py | 1 + metadata-ingestion/src/datahub/ingestion/graph/client.py | 2 +- 7 files changed, 8 insertions(+), 1 deletion(-) diff --git a/metadata-ingestion/examples/library/dataset_add_column_tag.py b/metadata-ingestion/examples/library/dataset_add_column_tag.py index 934ead98b7c7a8..db98c293b48e56 100644 --- a/metadata-ingestion/examples/library/dataset_add_column_tag.py +++ b/metadata-ingestion/examples/library/dataset_add_column_tag.py @@ -52,6 +52,7 @@ def get_simple_field_path_from_v2_field_path(field_path: str) -> str: entity_urn=dataset_urn, aspect="editableSchemaMetadata", aspect_type=EditableSchemaMetadataClass, + aspect_type_name=None, ) diff --git a/metadata-ingestion/examples/library/dataset_add_column_term.py b/metadata-ingestion/examples/library/dataset_add_column_term.py index c39c38bf51f1b5..7550d56ffbac8e 100644 --- a/metadata-ingestion/examples/library/dataset_add_column_term.py +++ b/metadata-ingestion/examples/library/dataset_add_column_term.py @@ -52,6 +52,7 @@ def get_simple_field_path_from_v2_field_path(field_path: str) -> str: entity_urn=dataset_urn, aspect="editableSchemaMetadata", aspect_type=EditableSchemaMetadataClass, + aspect_type_name=None, ) diff --git a/metadata-ingestion/examples/library/dataset_add_documentation.py b/metadata-ingestion/examples/library/dataset_add_documentation.py index c31c63163a69f5..23a801c9e7044e 100644 --- a/metadata-ingestion/examples/library/dataset_add_documentation.py +++ b/metadata-ingestion/examples/library/dataset_add_documentation.py @@ -44,6 +44,7 @@ entity_urn=dataset_urn, aspect="editableDatasetProperties", aspect_type=EditableDatasetPropertiesClass, + aspect_type_name=None, ) need_write = False @@ -77,6 +78,7 @@ entity_urn=dataset_urn, aspect="institutionalMemory", aspect_type=InstitutionalMemoryClass, + aspect_type_name=None, ) need_write = False diff --git a/metadata-ingestion/examples/library/dataset_add_owner.py b/metadata-ingestion/examples/library/dataset_add_owner.py index 6dc01da9aad2d6..2a273f01909c44 100644 --- a/metadata-ingestion/examples/library/dataset_add_owner.py +++ b/metadata-ingestion/examples/library/dataset_add_owner.py @@ -38,6 +38,7 @@ entity_urn=dataset_urn, aspect="ownership", aspect_type=OwnershipClass, + aspect_type_name=None, ) diff --git a/metadata-ingestion/examples/library/dataset_add_tag.py b/metadata-ingestion/examples/library/dataset_add_tag.py index 29503f90c48512..ede36f8966ff99 100644 --- a/metadata-ingestion/examples/library/dataset_add_tag.py +++ b/metadata-ingestion/examples/library/dataset_add_tag.py @@ -28,6 +28,7 @@ entity_urn=dataset_urn, aspect="globalTags", aspect_type=GlobalTagsClass, + aspect_type_name=None, ) tag_to_add = make_tag_urn("purchase") diff --git a/metadata-ingestion/examples/library/dataset_add_term.py b/metadata-ingestion/examples/library/dataset_add_term.py index 3976359d6d1d41..045e8634e5b112 100644 --- a/metadata-ingestion/examples/library/dataset_add_term.py +++ b/metadata-ingestion/examples/library/dataset_add_term.py @@ -29,6 +29,7 @@ entity_urn=dataset_urn, aspect="glossaryTerms", aspect_type=GlossaryTermsClass, + aspect_type_name=None, ) term_to_add = make_term_urn("Classification.HighlyConfidential") diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index be2a3be55a306f..10c8ed4bfc619c 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -94,8 +94,8 @@ def get_aspect( self, entity_urn: str, aspect: str, + aspect_type_name: Optional[str], aspect_type: Type[Aspect], - aspect_type_name: Optional[str] = None, ) -> Optional[Aspect]: url = f"{self._gms_server}/aspects/{urllib.parse.quote(entity_urn)}?aspect={aspect}&version=0" response = self._session.get(url) From b720f098f58da15b67f6467f66166695d579cf5f Mon Sep 17 00:00:00 2001 From: Shirshanka Das Date: Tue, 1 Mar 2022 21:28:59 -0800 Subject: [PATCH 2/2] adding new method and migrating new callers to call new method --- .../library/dataset_add_column_tag.py | 3 +- .../library/dataset_add_column_term.py | 3 +- .../library/dataset_add_documentation.py | 6 +-- .../examples/library/dataset_add_owner.py | 3 +- .../examples/library/dataset_add_tag.py | 3 +- .../examples/library/dataset_add_term.py | 3 +- metadata-ingestion/setup.py | 2 + .../src/datahub/ingestion/graph/client.py | 38 ++++++++++++++++--- 8 files changed, 41 insertions(+), 20 deletions(-) diff --git a/metadata-ingestion/examples/library/dataset_add_column_tag.py b/metadata-ingestion/examples/library/dataset_add_column_tag.py index db98c293b48e56..f5243ce28a5f01 100644 --- a/metadata-ingestion/examples/library/dataset_add_column_tag.py +++ b/metadata-ingestion/examples/library/dataset_add_column_tag.py @@ -48,11 +48,10 @@ def get_simple_field_path_from_v2_field_path(field_path: str) -> str: graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) -current_editable_schema_metadata = graph.get_aspect( +current_editable_schema_metadata = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="editableSchemaMetadata", aspect_type=EditableSchemaMetadataClass, - aspect_type_name=None, ) diff --git a/metadata-ingestion/examples/library/dataset_add_column_term.py b/metadata-ingestion/examples/library/dataset_add_column_term.py index 7550d56ffbac8e..ff1cad48a9f0c0 100644 --- a/metadata-ingestion/examples/library/dataset_add_column_term.py +++ b/metadata-ingestion/examples/library/dataset_add_column_term.py @@ -48,11 +48,10 @@ def get_simple_field_path_from_v2_field_path(field_path: str) -> str: graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) -current_editable_schema_metadata = graph.get_aspect( +current_editable_schema_metadata = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="editableSchemaMetadata", aspect_type=EditableSchemaMetadataClass, - aspect_type_name=None, ) diff --git a/metadata-ingestion/examples/library/dataset_add_documentation.py b/metadata-ingestion/examples/library/dataset_add_documentation.py index 23a801c9e7044e..da490df9bc4f76 100644 --- a/metadata-ingestion/examples/library/dataset_add_documentation.py +++ b/metadata-ingestion/examples/library/dataset_add_documentation.py @@ -40,11 +40,10 @@ gms_endpoint = "http://localhost:8080" graph = DataHubGraph(config=DatahubClientConfig(server=gms_endpoint)) -current_editable_properties = graph.get_aspect( +current_editable_properties = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="editableDatasetProperties", aspect_type=EditableDatasetPropertiesClass, - aspect_type_name=None, ) need_write = False @@ -74,11 +73,10 @@ log.info("Documentation already exists and is identical, omitting write") -current_institutional_memory = graph.get_aspect( +current_institutional_memory = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="institutionalMemory", aspect_type=InstitutionalMemoryClass, - aspect_type_name=None, ) need_write = False diff --git a/metadata-ingestion/examples/library/dataset_add_owner.py b/metadata-ingestion/examples/library/dataset_add_owner.py index 2a273f01909c44..d5eaf266423fe7 100644 --- a/metadata-ingestion/examples/library/dataset_add_owner.py +++ b/metadata-ingestion/examples/library/dataset_add_owner.py @@ -34,11 +34,10 @@ graph = DataHubGraph(DatahubClientConfig(server=gms_endpoint)) -current_owners: Optional[OwnershipClass] = graph.get_aspect( +current_owners: Optional[OwnershipClass] = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="ownership", aspect_type=OwnershipClass, - aspect_type_name=None, ) diff --git a/metadata-ingestion/examples/library/dataset_add_tag.py b/metadata-ingestion/examples/library/dataset_add_tag.py index ede36f8966ff99..2788bc16a815c2 100644 --- a/metadata-ingestion/examples/library/dataset_add_tag.py +++ b/metadata-ingestion/examples/library/dataset_add_tag.py @@ -24,11 +24,10 @@ dataset_urn = make_dataset_urn(platform="hive", name="realestate_db.sales", env="PROD") -current_tags: Optional[GlobalTagsClass] = graph.get_aspect( +current_tags: Optional[GlobalTagsClass] = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="globalTags", aspect_type=GlobalTagsClass, - aspect_type_name=None, ) tag_to_add = make_tag_urn("purchase") diff --git a/metadata-ingestion/examples/library/dataset_add_term.py b/metadata-ingestion/examples/library/dataset_add_term.py index 045e8634e5b112..1429a08a34b761 100644 --- a/metadata-ingestion/examples/library/dataset_add_term.py +++ b/metadata-ingestion/examples/library/dataset_add_term.py @@ -25,11 +25,10 @@ dataset_urn = make_dataset_urn(platform="hive", name="realestate_db.sales", env="PROD") -current_terms: Optional[GlossaryTermsClass] = graph.get_aspect( +current_terms: Optional[GlossaryTermsClass] = graph.get_aspect_v2( entity_urn=dataset_urn, aspect="glossaryTerms", aspect_type=GlossaryTermsClass, - aspect_type_name=None, ) term_to_add = make_term_urn("Classification.HighlyConfidential") diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index e089d610f93280..188847de5a80e1 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -52,6 +52,8 @@ def get_long_description(): # Pinning it to a version which works even though we are not using explicitly # https://github.com/aws/aws-sam-cli/issues/3661 "markupsafe==2.0.1", + "Deprecated", + "types-Deprecated", } kafka_common = { diff --git a/metadata-ingestion/src/datahub/ingestion/graph/client.py b/metadata-ingestion/src/datahub/ingestion/graph/client.py index 10c8ed4bfc619c..7345a900e5439b 100644 --- a/metadata-ingestion/src/datahub/ingestion/graph/client.py +++ b/metadata-ingestion/src/datahub/ingestion/graph/client.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List, Optional, Type from avro.schema import RecordSchema +from deprecated import deprecated from requests.adapters import Response from requests.models import HTTPError @@ -90,6 +91,9 @@ def _guess_entity_type(urn: str) -> str: assert urn.startswith("urn:li:"), "urns must start with urn:li:" return urn.split(":")[2] + @deprecated( + reason="Use get_aspect_v2 instead which makes aspect_type_name truly optional" + ) def get_aspect( self, entity_urn: str, @@ -97,6 +101,31 @@ def get_aspect( aspect_type_name: Optional[str], aspect_type: Type[Aspect], ) -> Optional[Aspect]: + return self.get_aspect_v2( + entity_urn=entity_urn, + aspect=aspect, + aspect_type=aspect_type, + aspect_type_name=aspect_type_name, + ) + + def get_aspect_v2( + self, + entity_urn: str, + aspect_type: Type[Aspect], + aspect: str, + aspect_type_name: Optional[str] = None, + ) -> Optional[Aspect]: + """ + Get an aspect for an entity. + + :param str entity_urn: The urn of the entity + :param Type[Aspect] aspect_type: The type class of the aspect being requested (e.g. datahub.metadata.schema_classes.DatasetProperties) + :param str aspect: The name of the aspect being requested (e.g. schemaMetadata, datasetProperties, etc.) + :param Optional[str] aspect_type_name: The fully qualified classname of the aspect being requested. Typically not needed and extracted automatically from the class directly. (e.g. com.linkedin.common.DatasetProperties) + :return: the Aspect as a dictionary if present, None if no aspect was found (HTTP status 404) + :rtype: Optional[Aspect] + :raises HttpError: if the HTTP response is not a 200 or a 404 + """ url = f"{self._gms_server}/aspects/{urllib.parse.quote(entity_urn)}?aspect={aspect}&version=0" response = self._session.get(url) if response.status_code == 404: @@ -126,26 +155,23 @@ def get_config(self) -> Dict[str, Any]: return self._get_generic(f"{self.config.server}/config") def get_ownership(self, entity_urn: str) -> Optional[OwnershipClass]: - return self.get_aspect( + return self.get_aspect_v2( entity_urn=entity_urn, aspect="ownership", - aspect_type_name="com.linkedin.common.Ownership", aspect_type=OwnershipClass, ) def get_tags(self, entity_urn: str) -> Optional[GlobalTagsClass]: - return self.get_aspect( + return self.get_aspect_v2( entity_urn=entity_urn, aspect="globalTags", - aspect_type_name="com.linkedin.common.GlobalTags", aspect_type=GlobalTagsClass, ) def get_glossary_terms(self, entity_urn: str) -> Optional[GlossaryTermsClass]: - return self.get_aspect( + return self.get_aspect_v2( entity_urn=entity_urn, aspect="glossaryTerms", - aspect_type_name="com.linkedin.common.GlossaryTerms", aspect_type=GlossaryTermsClass, )