diff --git a/databuilder/extractor/neptune_search_data_extractor.py b/databuilder/extractor/neptune_search_data_extractor.py index 4c1f01b1f..d6c1bec9b 100644 --- a/databuilder/extractor/neptune_search_data_extractor.py +++ b/databuilder/extractor/neptune_search_data_extractor.py @@ -142,7 +142,7 @@ def _user_search_query(graph: GraphTraversalSource, tag_filter: str) -> List[Dic def _dashboard_search_query(graph: GraphTraversalSource, tag_filter: str) -> List[Dict]: traversal = graph.V().hasLabel(DashboardMetadata.DASHBOARD_NODE_LABEL) - traversal = traversal.has('full_name') + traversal = traversal.has('name') if tag_filter: traversal = traversal.where('published_tag', tag_filter) @@ -184,13 +184,16 @@ def _dashboard_search_query(graph: GraphTraversalSource, tag_filter: str) -> Lis __.constant('') )) # group_description traversal = traversal.by( - __.out(DashboardMetadata.DASHBOARD_DASHBOARD_GROUP_RELATION_TYPE).values('group_url') + __.out(DashboardMetadata.DASHBOARD_DASHBOARD_GROUP_RELATION_TYPE).values('dashboard_group_url') ) # group_url traversal = traversal.by('dashboard_url') # dashboard_url traversal = traversal.by('key') # uri traversal = traversal.by( - __.out('EXECUTED').has('key', TextP.endingWith('_last_successful_execution')).values('timestamp') + __.coalesce( + __.out('EXECUTED').has('key', TextP.endingWith('_last_successful_execution')).values('timestamp'), + __.constant('') + ) ) # last_successful_run_timestamp traversal = traversal.by( __.out(DashboardQuery.DASHBOARD_QUERY_RELATION_TYPE).values('name').dedup().fold() diff --git a/databuilder/models/es_last_updated.py b/databuilder/models/es_last_updated.py index dc8853438..c46a27079 100644 --- a/databuilder/models/es_last_updated.py +++ b/databuilder/models/es_last_updated.py @@ -16,7 +16,7 @@ class ESLastUpdated(GraphSerializable): LABEL = 'Updatedtimestamp' KEY = 'amundsen_updated_timestamp' - LATEST_TIMESTAMP = 'latest_timestmap' + LATEST_TIMESTAMP = 'latest_timestamp' def __init__(self, timestamp: int, diff --git a/databuilder/models/table_metadata.py b/databuilder/models/table_metadata.py index 1b76afa83..bc33441c9 100644 --- a/databuilder/models/table_metadata.py +++ b/databuilder/models/table_metadata.py @@ -157,7 +157,7 @@ class ColumnMetadata: COLUMN_NODE_LABEL = 'Column' COLUMN_KEY_FORMAT = '{db}://{cluster}.{schema}/{tbl}/{col}' COLUMN_NAME = 'name' - COLUMN_TYPE = 'type' + COLUMN_TYPE = 'col_type' COLUMN_ORDER = 'sort_order' COLUMN_DESCRIPTION = 'description' COLUMN_DESCRIPTION_FORMAT = '{db}://{cluster}.{schema}/{tbl}/{col}/{description_id}' diff --git a/databuilder/models/table_stats.py b/databuilder/models/table_stats.py index ea3e73581..8a8e6d9d0 100644 --- a/databuilder/models/table_stats.py +++ b/databuilder/models/table_stats.py @@ -15,7 +15,7 @@ class TableColumnStats(GraphSerializable): """ LABEL = 'Stat' KEY_FORMAT = '{db}://{cluster}.{schema}' \ - '/{table}/{col}/{stat_name}/' + '/{table}/{col}/{stat_type}/' STAT_Column_RELATION_TYPE = 'STAT_OF' Column_STAT_RELATION_TYPE = 'STAT' @@ -40,7 +40,7 @@ def __init__(self, self.start_epoch = start_epoch self.end_epoch = end_epoch self.cluster = cluster - self.stat_name = stat_name + self.stat_type = stat_name self.stat_val = str(stat_val) self._node_iter = iter(self.create_nodes()) self._relation_iter = iter(self.create_relation()) @@ -64,7 +64,7 @@ def get_table_stat_model_key(self) -> str: schema=self.schema, table=self.table, col=self.col_name, - stat_name=self.stat_name) + stat_type=self.stat_type) def get_col_key(self) -> str: # no cluster, schema info from the input @@ -84,7 +84,7 @@ def create_nodes(self) -> List[GraphNode]: label=TableColumnStats.LABEL, attributes={ 'stat_val': self.stat_val, - 'stat_name': self.stat_name, + 'stat_type': self.stat_type, 'start_epoch': self.start_epoch, 'end_epoch': self.end_epoch, } diff --git a/databuilder/serializers/neptune_serializer.py b/databuilder/serializers/neptune_serializer.py index 7791864a7..1439a99a0 100644 --- a/databuilder/serializers/neptune_serializer.py +++ b/databuilder/serializers/neptune_serializer.py @@ -17,6 +17,7 @@ NEPTUNE_RELATIONSHIP_HEADER_TO = "~to" +METADATA_KEY_PROPERTY_NAME = 'key:String(single)' # last seen property names NEPTUNE_LAST_EXTRACTED_AT_NODE_PROPERTY_NAME = "last_extracted_datetime" NEPTUNE_LAST_EXTRACTED_AT_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT = "{name}:Date(single)".format( @@ -46,30 +47,33 @@ def convert_relationship(relationship: Optional[GraphRelationship]) -> List[Dict if relationship.start_key == '' or relationship.end_key == '': return [] - relation_id = "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=relationship.start_key, - to_vertex_id=relationship.end_key, - label=relationship.type + neptune_start_key = "{label}:{key}".format( + label=relationship.start_label, + key=relationship.start_key ) - relation_id_reverse = "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=relationship.end_key, - to_vertex_id=relationship.start_key, - label=relationship.reverse_type + neptune_end_key = "{label}:{key}".format( + label=relationship.end_label, + key=relationship.end_key ) + relation_id = get_forward_relationship_id(relationship) + relation_id_reverse = get_reverse_relationship_id(relationship) current_string_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') forward_relationship_doc = { NEPTUNE_HEADER_ID: relation_id, - NEPTUNE_RELATIONSHIP_HEADER_FROM: relationship.start_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: relationship.end_key, + METADATA_KEY_PROPERTY_NAME: relation_id, + NEPTUNE_RELATIONSHIP_HEADER_FROM: neptune_start_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: neptune_end_key, NEPTUNE_HEADER_LABEL: relationship.type, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: current_string_time, - NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, + } reverse_relationship_doc = { NEPTUNE_HEADER_ID: relation_id_reverse, - NEPTUNE_RELATIONSHIP_HEADER_FROM: relationship.end_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: relationship.start_key, + METADATA_KEY_PROPERTY_NAME: relation_id_reverse, + NEPTUNE_RELATIONSHIP_HEADER_FROM: neptune_end_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: neptune_start_key, NEPTUNE_HEADER_LABEL: relationship.reverse_type, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: current_string_time, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -90,6 +94,26 @@ def convert_relationship(relationship: Optional[GraphRelationship]) -> List[Dict ] +def get_forward_relationship_id(relationship: GraphRelationship) -> str: + return "{label}:{from_vertex_label}:{from_vertex_id}_{to_vertex_label}:{to_vertex_id}".format( + from_vertex_id=relationship.start_key, + from_vertex_label=relationship.start_label, + to_vertex_id=relationship.end_key, + to_vertex_label=relationship.end_label, + label=relationship.type + ) + + +def get_reverse_relationship_id(relationship: GraphRelationship) -> str: + return "{label}:{from_vertex_label}:{from_vertex_id}_{to_vertex_label}:{to_vertex_id}".format( + to_vertex_id=relationship.start_key, + to_vertex_label=relationship.start_label, + from_vertex_id=relationship.end_key, + from_vertex_label=relationship.end_label, + label=relationship.reverse_type + ) + + def convert_node(node: Optional[GraphNode]) -> Dict[str, Any]: if node is None: return {} @@ -98,8 +122,10 @@ def convert_node(node: Optional[GraphNode]) -> Dict[str, Any]: return {} current_string_time = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S') + node_id = get_node_id(node) node_dict = { - NEPTUNE_HEADER_ID: node.key, + NEPTUNE_HEADER_ID: node_id, + METADATA_KEY_PROPERTY_NAME: node_id, NEPTUNE_HEADER_LABEL: node.label, NEPTUNE_LAST_EXTRACTED_AT_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: current_string_time, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -117,6 +143,13 @@ def convert_node(node: Optional[GraphNode]) -> Dict[str, Any]: return node_dict +def get_node_id(node: GraphNode) -> str: + return "{label}:{key}".format( + label=node.label, + key=node.key + ) + + def _get_neptune_type_for_value(value: Any) -> Optional[str]: if isinstance(value, six.string_types): return "String" diff --git a/tests/unit/models/dashboard/test_dashboard_chart.py b/tests/unit/models/dashboard/test_dashboard_chart.py index 5e6c8dc07..7ae75337b 100644 --- a/tests/unit/models/dashboard/test_dashboard_chart.py +++ b/tests/unit/models/dashboard/test_dashboard_chart.py @@ -12,7 +12,7 @@ ) from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -44,7 +44,8 @@ def test_create_nodes(self) -> None: 'LABEL': 'Chart' } neptune_expected = { - '~id': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', + '~id': 'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', + METADATA_KEY_PROPERTY_NAME: 'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', '~label': 'Chart', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -77,7 +78,8 @@ def test_create_nodes(self) -> None: 'url': 'http://gold.foo.bar/' } neptune_expected2 = { - '~id': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', + '~id': 'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', + METADATA_KEY_PROPERTY_NAME: 'Chart:_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', '~label': 'Chart', 'id:String(single)': 'c_id', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, @@ -112,26 +114,36 @@ def test_create_relation(self) -> None: } neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=start_key, - to_vertex_id=end_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Query:" + start_key, + to_vertex_id="Chart:" + end_key, label='HAS_CHART' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: end_key, + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Query:" + start_key, + to_vertex_id="Chart:" + end_key, + label='HAS_CHART' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: "Query:" + start_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: "Chart:" + end_key, NEPTUNE_HEADER_LABEL: 'HAS_CHART', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=end_key, - to_vertex_id=start_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Chart:" + end_key, + to_vertex_id="Query:" + start_key, + label='CHART_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Chart:" + end_key, + to_vertex_id="Query:" + start_key, label='CHART_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: end_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, + NEPTUNE_RELATIONSHIP_HEADER_FROM: "Chart:" + end_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: "Query:" + start_key, NEPTUNE_HEADER_LABEL: 'CHART_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/dashboard/test_dashboard_last_modified.py b/tests/unit/models/dashboard/test_dashboard_last_modified.py index 09e26cce6..244abb257 100644 --- a/tests/unit/models/dashboard/test_dashboard_last_modified.py +++ b/tests/unit/models/dashboard/test_dashboard_last_modified.py @@ -12,7 +12,7 @@ ) from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -55,7 +55,8 @@ def test_neptune_dashboard_timestamp_nodes(self) -> None: actual = self.dashboard_last_modified.create_next_node() actual_neptune_serialized = neptune_serializer.convert_node(actual) neptune_expected = { - NEPTUNE_HEADER_ID: self.expected_ts_key, + NEPTUNE_HEADER_ID: 'Timestamp:' + self.expected_ts_key, + METADATA_KEY_PROPERTY_NAME: 'Timestamp:' + self.expected_ts_key, NEPTUNE_HEADER_LABEL: 'Timestamp', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -87,31 +88,41 @@ def test_dashboard_owner_relations_neptune(self) -> None: actual = self.dashboard_last_modified.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.expected_dashboard_key, - to_vertex_id=self.expected_ts_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:' + self.expected_dashboard_key, + to_vertex_id='Timestamp:' + self.expected_ts_key, label='LAST_UPDATED_AT' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.expected_dashboard_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.expected_ts_key, + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:' + self.expected_dashboard_key, + to_vertex_id='Timestamp:' + self.expected_ts_key, + label='LAST_UPDATED_AT' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:' + self.expected_dashboard_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Timestamp:' + self.expected_ts_key, NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_AT', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.expected_ts_key, - to_vertex_id=self.expected_dashboard_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Timestamp:' + self.expected_ts_key, + to_vertex_id='Dashboard:' + self.expected_dashboard_key, label='LAST_UPDATED_TIME_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.expected_ts_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.expected_dashboard_key, + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Timestamp:' + self.expected_ts_key, + to_vertex_id='Dashboard:' + self.expected_dashboard_key, + label='LAST_UPDATED_TIME_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Timestamp:' + self.expected_ts_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:' + self.expected_dashboard_key, NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_TIME_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } - + self.maxDiff = None assert actual is not None self.assertDictEqual(actual_serialized[0], neptune_forward_expected) self.assertDictEqual(actual_serialized[1], neptune_reversed_expected) diff --git a/tests/unit/models/dashboard/test_dashboard_metadata.py b/tests/unit/models/dashboard/test_dashboard_metadata.py index a78099387..cf34bdf7f 100644 --- a/tests/unit/models/dashboard/test_dashboard_metadata.py +++ b/tests/unit/models/dashboard/test_dashboard_metadata.py @@ -9,7 +9,7 @@ from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -173,25 +173,35 @@ def test_full_dashboard_example_neptune(self) -> None: expected_neptune_rels = [ [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold', - to_vertex_id='_dashboard://gold.Product - Jobs.cz', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Cluster:_dashboard://gold', + to_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_GROUP' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Cluster:_dashboard://gold', + to_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + label='DASHBOARD_GROUP' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Cluster:_dashboard://gold', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DASHBOARD_GROUP', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz', - to_vertex_id='_dashboard://gold', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + to_vertex_id='Cluster:_dashboard://gold', + label='DASHBOARD_GROUP_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + to_vertex_id='Cluster:_dashboard://gold', label='DASHBOARD_GROUP_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Cluster:_dashboard://gold', NEPTUNE_HEADER_LABEL: 'DASHBOARD_GROUP_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -199,25 +209,35 @@ def test_full_dashboard_example_neptune(self) -> None: ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz', - to_vertex_id='_dashboard://gold.Product - Jobs.cz/_description', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + to_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/_description', + label='DESCRIPTION' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + to_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/_description', label='DESCRIPTION' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz/_description', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz/_description', - to_vertex_id='_dashboard://gold.Product - Jobs.cz', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/_description', + to_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DESCRIPTION_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/_description', + to_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + label='DESCRIPTION_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:_dashboard://gold.Product - Jobs.cz/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -225,25 +245,35 @@ def test_full_dashboard_example_neptune(self) -> None: ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', - to_vertex_id='_dashboard://gold.Product - Jobs.cz', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + label='DASHBOARD_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', label='DASHBOARD_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz/Agent', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'DASHBOARD_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz', - to_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + label='DASHBOARD' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DASHBOARD' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'DASHBOARD', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -251,25 +281,37 @@ def test_full_dashboard_example_neptune(self) -> None: ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', - to_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent/_description', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', label='DESCRIPTION' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz/Agent', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz/Agent/_description', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', + label='DESCRIPTION' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_RELATIONSHIP_HEADER_TO: + 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent/_description', - to_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + label='DESCRIPTION_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='DESCRIPTION_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz/Agent/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_RELATIONSHIP_HEADER_FROM: + 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -277,25 +319,35 @@ def test_full_dashboard_example_neptune(self) -> None: ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', - to_vertex_id='test_tag', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Tag:test_tag', + label='TAG' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Tag:test_tag', label='TAG' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz/Agent', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'test_tag', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Tag:test_tag', NEPTUNE_HEADER_LABEL: 'TAG', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='test_tag', - to_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Tag:test_tag', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'test_tag', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz/Agent', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Tag:test_tag', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + label='TAG_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Tag:test_tag', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'TAG_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -303,25 +355,35 @@ def test_full_dashboard_example_neptune(self) -> None: ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', - to_vertex_id='tag2', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Tag:tag2', + label='TAG' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + to_vertex_id='Tag:tag2', label='TAG' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.Product - Jobs.cz/Agent', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'tag2', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Tag:tag2', NEPTUNE_HEADER_LABEL: 'TAG', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='tag2', - to_vertex_id='_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Tag:tag2', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + label='TAG_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Tag:tag2', + to_vertex_id='Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', label='TAG_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'tag2', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Tag:tag2', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'TAG_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -331,14 +393,16 @@ def test_full_dashboard_example_neptune(self) -> None: expected_neptune_nodes = [ { - NEPTUNE_HEADER_ID: '_dashboard://gold', + NEPTUNE_HEADER_ID: 'Cluster:_dashboard://gold', + METADATA_KEY_PROPERTY_NAME: 'Cluster:_dashboard://gold', NEPTUNE_HEADER_LABEL: 'Cluster', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'name:String(single)': 'gold' }, { - NEPTUNE_HEADER_ID: '_dashboard://gold.Product - Jobs.cz/Agent', + NEPTUNE_HEADER_ID: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', + METADATA_KEY_PROPERTY_NAME: 'Dashboard:_dashboard://gold.Product - Jobs.cz/Agent', NEPTUNE_HEADER_LABEL: 'Dashboard', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -347,7 +411,8 @@ def test_full_dashboard_example_neptune(self) -> None: 'created_timestamp:Long(single)': 123456789, }, { - NEPTUNE_HEADER_ID: '_dashboard://gold.Product - Jobs.cz', + NEPTUNE_HEADER_ID: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', + METADATA_KEY_PROPERTY_NAME: 'Dashboardgroup:_dashboard://gold.Product - Jobs.cz', NEPTUNE_HEADER_LABEL: 'Dashboardgroup', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -355,34 +420,39 @@ def test_full_dashboard_example_neptune(self) -> None: 'dashboard_group_url:String(single)': 'https://foo.bar/dashboard_group/foo' }, { - NEPTUNE_HEADER_ID: '_dashboard://gold.Product - Jobs.cz/_description', + NEPTUNE_HEADER_ID: 'Description:_dashboard://gold.Product - Jobs.cz/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:_dashboard://gold.Product - Jobs.cz/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'description:String(single)': 'foo dashboard group description', }, { - NEPTUNE_HEADER_ID: '_dashboard://gold.Product - Jobs.cz/Agent/_description', + NEPTUNE_HEADER_ID: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:_dashboard://gold.Product - Jobs.cz/Agent/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'description:String(single)': 'Agent dashboard description' }, { - NEPTUNE_HEADER_ID: 'test_tag', + NEPTUNE_HEADER_ID: 'Tag:test_tag', + METADATA_KEY_PROPERTY_NAME: 'Tag:test_tag', NEPTUNE_HEADER_LABEL: 'Tag', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'tag_type:String(single)': 'dashboard' }, { - NEPTUNE_HEADER_ID: 'tag2', + NEPTUNE_HEADER_ID: 'Tag:tag2', + METADATA_KEY_PROPERTY_NAME: 'Tag:tag2', NEPTUNE_HEADER_LABEL: 'Tag', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'tag_type:String(single)': 'dashboard' }, ] + self.maxDiff = None node_row = self.full_dashboard_metadata.next_node() actual = [] while node_row: diff --git a/tests/unit/models/dashboard/test_dashboard_owner.py b/tests/unit/models/dashboard/test_dashboard_owner.py index 40b4238c1..7ff88b0dd 100644 --- a/tests/unit/models/dashboard/test_dashboard_owner.py +++ b/tests/unit/models/dashboard/test_dashboard_owner.py @@ -11,9 +11,10 @@ ) from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, - NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, - NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, + NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, + NEPTUNE_RELATIONSHIP_HEADER_TO, ) @@ -51,26 +52,38 @@ def test_dashboard_owner_relations_neptune(self) -> None: actual = self.dashboard_owner.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', - to_vertex_id='foo@bar.com', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + to_vertex_id='User:foo@bar.com', label='OWNER' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'foo@bar.com', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + to_vertex_id='User:foo@bar.com', + label='OWNER' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: + 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'User:foo@bar.com', NEPTUNE_HEADER_LABEL: 'OWNER', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='foo@bar.com', - to_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='User:foo@bar.com', + to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + label='OWNER_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='User:foo@bar.com', + to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='OWNER_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'foo@bar.com', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'User:foo@bar.com', + NEPTUNE_RELATIONSHIP_HEADER_TO: + 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_HEADER_LABEL: 'OWNER_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/dashboard/test_dashboard_query.py b/tests/unit/models/dashboard/test_dashboard_query.py index cfeb7edda..1f0e63525 100644 --- a/tests/unit/models/dashboard/test_dashboard_query.py +++ b/tests/unit/models/dashboard/test_dashboard_query.py @@ -11,7 +11,7 @@ ) from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -48,7 +48,8 @@ def test_create_nodes_neptune(self) -> None: actual = self.dashboard_query.create_next_node() actual_serialized = neptune_serializer.convert_node(actual) neptune_expected = { - NEPTUNE_HEADER_ID: '_dashboard://gold.dg_id/d_id/query/q_id', + NEPTUNE_HEADER_ID: 'Query:_dashboard://gold.dg_id/d_id/query/q_id', + METADATA_KEY_PROPERTY_NAME: 'Query:_dashboard://gold.dg_id/d_id/query/q_id', NEPTUNE_HEADER_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -77,26 +78,36 @@ def test_create_relation_neptune(self) -> None: actual = self.dashboard_query.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.dg_id/d_id', - to_vertex_id='_dashboard://gold.dg_id/d_id/query/q_id', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', + to_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', label='HAS_QUERY' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.dg_id/d_id', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.dg_id/d_id/query/q_id', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', + to_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', + label='HAS_QUERY' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Dashboard:_dashboard://gold.dg_id/d_id', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Query:_dashboard://gold.dg_id/d_id/query/q_id', NEPTUNE_HEADER_LABEL: 'HAS_QUERY', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='_dashboard://gold.dg_id/d_id/query/q_id', - to_vertex_id='_dashboard://gold.dg_id/d_id', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', + to_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', + label='QUERY_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Query:_dashboard://gold.dg_id/d_id/query/q_id', + to_vertex_id='Dashboard:_dashboard://gold.dg_id/d_id', label='QUERY_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: '_dashboard://gold.dg_id/d_id/query/q_id', - NEPTUNE_RELATIONSHIP_HEADER_TO: '_dashboard://gold.dg_id/d_id', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Query:_dashboard://gold.dg_id/d_id/query/q_id', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Dashboard:_dashboard://gold.dg_id/d_id', NEPTUNE_HEADER_LABEL: 'QUERY_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/dashboard/test_dashboard_table.py b/tests/unit/models/dashboard/test_dashboard_table.py index 42ce988d0..27d28715f 100644 --- a/tests/unit/models/dashboard/test_dashboard_table.py +++ b/tests/unit/models/dashboard/test_dashboard_table.py @@ -11,9 +11,10 @@ ) from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, - NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, - NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, + NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, + NEPTUNE_RELATIONSHIP_HEADER_TO, ) @@ -41,26 +42,38 @@ def test_dashboard_table_relations(self) -> None: RELATION_REVERSE_TYPE: 'TABLE_OF_DASHBOARD'} neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', - to_vertex_id='hive://gold.schema/table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + to_vertex_id='Table:hive://gold.schema/table1', label='DASHBOARD_WITH_TABLE' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.schema/table1', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + to_vertex_id='Table:hive://gold.schema/table1', + label='DASHBOARD_WITH_TABLE' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: + 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.schema/table1', NEPTUNE_HEADER_LABEL: 'DASHBOARD_WITH_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.schema/table1', - to_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.schema/table1', + to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + label='TABLE_OF_DASHBOARD' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.schema/table1', + to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', label='TABLE_OF_DASHBOARD' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.schema/table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.schema/table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: + 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', NEPTUNE_HEADER_LABEL: 'TABLE_OF_DASHBOARD', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/dashboard/test_dashboard_usage.py b/tests/unit/models/dashboard/test_dashboard_usage.py index a5c24dd00..177083889 100644 --- a/tests/unit/models/dashboard/test_dashboard_usage.py +++ b/tests/unit/models/dashboard/test_dashboard_usage.py @@ -12,9 +12,10 @@ ) from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, - NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, - NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, + NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, + NEPTUNE_RELATIONSHIP_HEADER_TO, ) @@ -84,14 +85,23 @@ def test_dashboard_owner_relations_neptune(self) -> None: actual = dashboard_usage.create_next_relation() actual_serialized = neptune_serializer.convert_relationship(actual) + forward_id = "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + to_vertex_id='User:foo@bar.com', + label='READ_BY' + ) + reverse_id = "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='User:foo@bar.com', + to_vertex_id='Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + label='READ' + ) + + dashboard_id = 'Dashboard:product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id' neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', - to_vertex_id='foo@bar.com', - label='READ_BY' - ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'foo@bar.com', + NEPTUNE_HEADER_ID: forward_id, + METADATA_KEY_PROPERTY_NAME: forward_id, + NEPTUNE_RELATIONSHIP_HEADER_FROM: dashboard_id, + NEPTUNE_RELATIONSHIP_HEADER_TO: 'User:foo@bar.com', NEPTUNE_HEADER_LABEL: 'READ_BY', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -99,13 +109,10 @@ def test_dashboard_owner_relations_neptune(self) -> None: } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='foo@bar.com', - to_vertex_id='product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', - label='READ' - ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'foo@bar.com', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id', + NEPTUNE_HEADER_ID: reverse_id, + METADATA_KEY_PROPERTY_NAME: reverse_id, + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'User:foo@bar.com', + NEPTUNE_RELATIONSHIP_HEADER_TO: dashboard_id, NEPTUNE_HEADER_LABEL: 'READ', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -113,6 +120,7 @@ def test_dashboard_owner_relations_neptune(self) -> None: } assert actual is not None + self.maxDiff = None self.assertDictEqual(neptune_forward_expected, actual_serialized[0]) self.assertDictEqual(neptune_reversed_expected, actual_serialized[1]) self.assertIsNone(dashboard_usage.create_next_relation()) diff --git a/tests/unit/models/schema/test_schema.py b/tests/unit/models/schema/test_schema.py index 948991f42..59b57c5b1 100644 --- a/tests/unit/models/schema/test_schema.py +++ b/tests/unit/models/schema/test_schema.py @@ -7,7 +7,7 @@ from databuilder.models.schema.schema import SchemaModel from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -40,7 +40,8 @@ def test_create_nodes(self) -> None: def test_create_nodes_neptune(self) -> None: schema_node = self.schema.create_next_node() expected_serialized_schema_node = { - NEPTUNE_HEADER_ID: 'db://cluster.schema', + NEPTUNE_HEADER_ID: 'Schema:db://cluster.schema', + METADATA_KEY_PROPERTY_NAME: 'Schema:db://cluster.schema', NEPTUNE_HEADER_LABEL: 'Schema', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -53,7 +54,8 @@ def test_create_nodes_neptune(self) -> None: ) schema_desc_node = self.schema.create_next_node() excepted_serialized_schema_desc_node = { - NEPTUNE_HEADER_ID: 'db://cluster.schema/_description', + NEPTUNE_HEADER_ID: 'Description:db://cluster.schema/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:db://cluster.schema/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -109,33 +111,38 @@ def test_create_relation(self) -> None: def test_create_relation_neptune(self) -> None: actual = self.schema.create_next_relation() serialized_actual = neptune_serializer.convert_relationship(actual) + forward_header_id = "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Schema:db://cluster.schema', + to_vertex_id='Description:db://cluster.schema/_description', + label='DESCRIPTION' + ) + reverse_header_id = "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:db://cluster.schema/_description', + to_vertex_id='Schema:db://cluster.schema', + label='DESCRIPTION_OF' + ) neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='db://cluster.schema', - to_vertex_id='db://cluster.schema/_description', - label='DESCRIPTION' - ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'db://cluster.schema', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'db://cluster.schema/_description', + NEPTUNE_HEADER_ID: forward_header_id, + METADATA_KEY_PROPERTY_NAME: forward_header_id, + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Schema:db://cluster.schema', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:db://cluster.schema/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='db://cluster.schema/_description', - to_vertex_id='db://cluster.schema', - label='DESCRIPTION_OF' - ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'db://cluster.schema/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'db://cluster.schema', + NEPTUNE_HEADER_ID: reverse_header_id, + METADATA_KEY_PROPERTY_NAME: reverse_header_id, + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:db://cluster.schema/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Schema:db://cluster.schema', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } + self.maxDiff = None self.assertDictEqual(serialized_actual[0], neptune_forward_expected) self.assertDictEqual(serialized_actual[1], neptune_reversed_expected) diff --git a/tests/unit/models/test_application.py b/tests/unit/models/test_application.py index b4f159838..e47511aa6 100644 --- a/tests/unit/models/test_application.py +++ b/tests/unit/models/test_application.py @@ -12,7 +12,7 @@ from databuilder.models.table_metadata import TableMetadata from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -56,8 +56,10 @@ def test_create_next_node(self) -> None: def test_create_next_node_neptune(self) -> None: next_node = self.application.create_next_node() serialized_next_node = neptune_serializer.convert_node(next_node) + node_id = 'Application:application://gold.airflow/event_test/hive.default.test_table' neptune_expected = { - NEPTUNE_HEADER_ID: 'application://gold.airflow/event_test/hive.default.test_table', + NEPTUNE_HEADER_ID: node_id, + METADATA_KEY_PROPERTY_NAME: node_id, NEPTUNE_HEADER_LABEL: 'Application', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -74,28 +76,39 @@ def test_create_next_relation(self) -> None: self.assertEquals(serialized_next_relation, self.expected_relation_result) def test_create_next_relation_neptune(self) -> None: - + application_id = 'Application:application://gold.airflow/event_test/hive.default.test_table' + table_id = 'Table:hive://gold.default/test_table' neptune_forward_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.default/test_table', - to_vertex_id='application://gold.airflow/event_test/hive.default.test_table', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=table_id, + to_vertex_id=application_id, + label='DERIVED_FROM' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=table_id, + to_vertex_id=application_id, label='DERIVED_FROM' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.default/test_table', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'application://gold.airflow/event_test/hive.default.test_table', + NEPTUNE_RELATIONSHIP_HEADER_FROM: table_id, + NEPTUNE_RELATIONSHIP_HEADER_TO: application_id, NEPTUNE_HEADER_LABEL: 'DERIVED_FROM', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='application://gold.airflow/event_test/hive.default.test_table', - to_vertex_id='hive://gold.default/test_table', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=application_id, + to_vertex_id=table_id, + label='GENERATES' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=application_id, + to_vertex_id=table_id, label='GENERATES' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'application://gold.airflow/event_test/hive.default.test_table', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.default/test_table', + NEPTUNE_RELATIONSHIP_HEADER_FROM: application_id, + NEPTUNE_RELATIONSHIP_HEADER_TO: table_id, NEPTUNE_HEADER_LABEL: 'GENERATES', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/test_badge.py b/tests/unit/models/test_badge.py index e04312324..b4947601f 100644 --- a/tests/unit/models/test_badge.py +++ b/tests/unit/models/test_badge.py @@ -12,7 +12,7 @@ ) from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -65,17 +65,19 @@ def test_create_nodes_neptune(self) -> None: neptune_serializer.convert_node(node) for node in nodes ] - + node_id_1 = BadgeMetadata.BADGE_NODE_LABEL + ":" + BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge1.name) expected_node1 = { - NEPTUNE_HEADER_ID: BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge1.name), + NEPTUNE_HEADER_ID: node_id_1, + METADATA_KEY_PROPERTY_NAME: node_id_1, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, BadgeMetadata.BADGE_CATEGORY + ':String(single)': badge1.category } - + node_id_2 = BadgeMetadata.BADGE_NODE_LABEL + ":" + BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge2.name) expected_node2 = { - NEPTUNE_HEADER_ID: BadgeMetadata.BADGE_KEY_FORMAT.format(badge=badge2.name), + NEPTUNE_HEADER_ID: node_id_2, + METADATA_KEY_PROPERTY_NAME: node_id_2, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -138,53 +140,77 @@ def test_create_relation_neptune(self) -> None: neptune_serializer.convert_relationship(rel) for rel in relations ], []) + badge_id_1 = BadgeMetadata.BADGE_NODE_LABEL + ':' + BadgeMetadata.get_badge_key(badge1.name) + badge_id_2 = BadgeMetadata.BADGE_NODE_LABEL + ':' + BadgeMetadata.get_badge_key(badge2.name) + start_key = self.badge_metada.start_label + ':' + self.badge_metada.start_key + neptune_forward_expected_1 = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.badge_metada.start_key, - to_vertex_id=BadgeMetadata.get_badge_key(badge1.name), + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=start_key, + to_vertex_id=badge_id_1, label=BadgeMetadata.BADGE_RELATION_TYPE, ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.badge_metada.start_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: BadgeMetadata.get_badge_key(badge1.name), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=start_key, + to_vertex_id=badge_id_1, + label=BadgeMetadata.BADGE_RELATION_TYPE, + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: badge_id_1, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected_1 = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=BadgeMetadata.get_badge_key(badge1.name), - to_vertex_id=self.badge_metada.start_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=badge_id_1, + to_vertex_id=start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: BadgeMetadata.get_badge_key(badge1.name), - NEPTUNE_RELATIONSHIP_HEADER_TO: self.badge_metada.start_key, + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=badge_id_1, + to_vertex_id=start_key, + label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: badge_id_1, + NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_forward_expected_2 = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.badge_metada.start_key, - to_vertex_id=BadgeMetadata.get_badge_key(badge2.name), + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=start_key, + to_vertex_id=badge_id_2, + label=BadgeMetadata.BADGE_RELATION_TYPE, + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=start_key, + to_vertex_id=badge_id_2, label=BadgeMetadata.BADGE_RELATION_TYPE, ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.badge_metada.start_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: BadgeMetadata.get_badge_key(badge2.name), + NEPTUNE_RELATIONSHIP_HEADER_FROM: start_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: badge_id_2, NEPTUNE_HEADER_LABEL: BadgeMetadata.BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB } neptune_reversed_expected_2 = { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=BadgeMetadata.get_badge_key(badge2.name), - to_vertex_id=self.badge_metada.start_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=badge_id_2, + to_vertex_id=start_key, + label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=badge_id_2, + to_vertex_id=start_key, label=BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: BadgeMetadata.get_badge_key(badge2.name), - NEPTUNE_RELATIONSHIP_HEADER_TO: self.badge_metada.start_key, + NEPTUNE_RELATIONSHIP_HEADER_FROM: badge_id_2, + NEPTUNE_RELATIONSHIP_HEADER_TO: start_key, NEPTUNE_HEADER_LABEL: BadgeMetadata.INVERSE_BADGE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/test_es_last_updated.py b/tests/unit/models/test_es_last_updated.py index efe42304d..828db7526 100644 --- a/tests/unit/models/test_es_last_updated.py +++ b/tests/unit/models/test_es_last_updated.py @@ -17,7 +17,7 @@ def setUp(self) -> None: self.expected_node_result = { NODE_KEY: 'amundsen_updated_timestamp', NODE_LABEL: 'Updatedtimestamp', - 'latest_timestmap:UNQUOTED': 100, + 'latest_timestamp:UNQUOTED': 100, } def test_create_nodes(self) -> None: diff --git a/tests/unit/models/test_fixtures/table_metadata_fixtures.py b/tests/unit/models/test_fixtures/table_metadata_fixtures.py index 120a906f8..a8004998c 100644 --- a/tests/unit/models/test_fixtures/table_metadata_fixtures.py +++ b/tests/unit/models/test_fixtures/table_metadata_fixtures.py @@ -4,7 +4,7 @@ from unittest.mock import ANY from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -13,7 +13,8 @@ EXPECTED_NEPTUNE_NODES = [ { 'name:String(single)': 'test_table1', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: 'Table:hive://gold.test_schema1/test_table1', + METADATA_KEY_PROPERTY_NAME: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'Table', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -21,7 +22,8 @@ }, { 'description:String(single)': 'test_table1', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/_description', + NEPTUNE_HEADER_ID: 'Description:hive://gold.test_schema1/test_table1/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:hive://gold.test_schema1/test_table1/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -29,16 +31,18 @@ }, { 'sort_order:Long(single)': 0, - 'type:String(single)': 'bigint', + 'col_type:String(single)': 'bigint', 'name:String(single)': 'test_id1', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/test_id1', + NEPTUNE_HEADER_ID: 'Column:hive://gold.test_schema1/test_table1/test_id1', + METADATA_KEY_PROPERTY_NAME: 'Column:hive://gold.test_schema1/test_table1/test_id1', NEPTUNE_HEADER_LABEL: 'Column' }, { 'description:String(single)': 'description of test_table1', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/test_id1/_description', + NEPTUNE_HEADER_ID: 'Description:hive://gold.test_schema1/test_table1/test_id1/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:hive://gold.test_schema1/test_table1/test_id1/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -46,16 +50,18 @@ }, { 'sort_order:Long(single)': 1, - 'type:String(single)': 'bigint', + 'col_type:String(single)': 'bigint', 'name:String(single)': 'test_id2', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/test_id2', + NEPTUNE_HEADER_ID: 'Column:hive://gold.test_schema1/test_table1/test_id2', + METADATA_KEY_PROPERTY_NAME: 'Column:hive://gold.test_schema1/test_table1/test_id2', NEPTUNE_HEADER_LABEL: 'Column', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, }, { 'description:String(single)': 'description of test_id2', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/test_id2/_description', + NEPTUNE_HEADER_ID: 'Description:hive://gold.test_schema1/test_table1/test_id2/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:hive://gold.test_schema1/test_table1/test_id2/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -63,25 +69,28 @@ }, { 'sort_order:Long(single)': 2, - 'type:String(single)': 'boolean', + 'col_type:String(single)': 'boolean', 'name:String(single)': 'is_active', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/is_active', + NEPTUNE_HEADER_ID: 'Column:hive://gold.test_schema1/test_table1/is_active', + METADATA_KEY_PROPERTY_NAME: 'Column:hive://gold.test_schema1/test_table1/is_active', NEPTUNE_HEADER_LABEL: 'Column', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, }, { 'sort_order:Long(single)': 3, - 'type:String(single)': 'varchar', + 'col_type:String(single)': 'varchar', 'name:String(single)': 'source', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/source', + NEPTUNE_HEADER_ID: 'Column:hive://gold.test_schema1/test_table1/source', + METADATA_KEY_PROPERTY_NAME: 'Column:hive://gold.test_schema1/test_table1/source', NEPTUNE_HEADER_LABEL: 'Column', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, }, { 'description:String(single)': 'description of source', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/source/_description', + NEPTUNE_HEADER_ID: 'Description:hive://gold.test_schema1/test_table1/source/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:hive://gold.test_schema1/test_table1/source/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -89,16 +98,18 @@ }, { 'sort_order:Long(single)': 4, - 'type:String(single)': 'timestamp', + 'col_type:String(single)': 'timestamp', 'name:String(single)': 'etl_created_at', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/etl_created_at', + NEPTUNE_HEADER_ID: 'Column:hive://gold.test_schema1/test_table1/etl_created_at', + METADATA_KEY_PROPERTY_NAME: 'Column:hive://gold.test_schema1/test_table1/etl_created_at', NEPTUNE_HEADER_LABEL: 'Column', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, }, { 'description:String(single)': 'description of etl_created_at', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/etl_created_at/_description', + NEPTUNE_HEADER_ID: 'Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', + METADATA_KEY_PROPERTY_NAME: 'Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', NEPTUNE_HEADER_LABEL: 'Description', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -106,30 +117,34 @@ }, { 'sort_order:Long(single)': 5, - 'type:String(single)': 'varchar', + 'col_type:String(single)': 'varchar', 'name:String(single)': 'ds', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1/test_table1/ds', + NEPTUNE_HEADER_ID: 'Column:hive://gold.test_schema1/test_table1/ds', + METADATA_KEY_PROPERTY_NAME: 'Column:hive://gold.test_schema1/test_table1/ds', NEPTUNE_HEADER_LABEL: 'Column', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, }, { 'name:String(single)': 'hive', - NEPTUNE_HEADER_ID: 'database://hive', + NEPTUNE_HEADER_ID: 'Database:database://hive', + METADATA_KEY_PROPERTY_NAME: 'Database:database://hive', NEPTUNE_HEADER_LABEL: 'Database', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, }, { 'name:String(single)': 'gold', - NEPTUNE_HEADER_ID: 'hive://gold', + NEPTUNE_HEADER_ID: 'Cluster:hive://gold', + METADATA_KEY_PROPERTY_NAME: 'Cluster:hive://gold', NEPTUNE_HEADER_LABEL: 'Cluster', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, }, { 'name:String(single)': 'test_schema1', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1', + NEPTUNE_HEADER_ID: 'Schema:hive://gold.test_schema1', + METADATA_KEY_PROPERTY_NAME: 'Schema:hive://gold.test_schema1', NEPTUNE_HEADER_LABEL: 'Schema', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -139,25 +154,35 @@ EXPECTED_RELATIONSHIPS_NEPTUNE = [ [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Schema:hive://gold.test_schema1', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='TABLE' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Schema:hive://gold.test_schema1', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='TABLE' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Schema:hive://gold.test_schema1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Schema:hive://gold.test_schema1', + label='TABLE_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Schema:hive://gold.test_schema1', label='TABLE_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Schema:hive://gold.test_schema1', NEPTUNE_HEADER_LABEL: 'TABLE_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -165,25 +190,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1/test_table1/_description', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/_description', + label='DESCRIPTION' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/_description', label='DESCRIPTION' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/_description', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:hive://gold.test_schema1/test_table1/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/_description', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/_description', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='DESCRIPTION_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/_description', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='DESCRIPTION_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:hive://gold.test_schema1/test_table1/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -191,25 +226,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1/test_table1/test_id1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', + label='COLUMN' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', label='COLUMN' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/test_id1', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/test_id1', NEPTUNE_HEADER_LABEL: 'COLUMN', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/test_id1', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='COLUMN_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='COLUMN_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/test_id1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/test_id1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'COLUMN_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -217,25 +262,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/test_id1', - to_vertex_id='hive://gold.test_schema1/test_table1/test_id1/_description', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id1/_description', label='DESCRIPTION' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/test_id1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/test_id1/_description', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id1/_description', + label='DESCRIPTION' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/test_id1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:hive://gold.test_schema1/test_table1/test_id1/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/test_id1/_description', - to_vertex_id='hive://gold.test_schema1/test_table1/test_id1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id1/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', + label='DESCRIPTION_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id1/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id1', label='DESCRIPTION_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/test_id1/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/test_id1', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:hive://gold.test_schema1/test_table1/test_id1/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/test_id1', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -243,25 +298,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1/test_table1/test_id2', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', + label='COLUMN' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', label='COLUMN' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/test_id2', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/test_id2', NEPTUNE_HEADER_LABEL: 'COLUMN', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/test_id2', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='COLUMN_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/test_id2', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='COLUMN_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/test_id2', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'COLUMN_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -269,25 +334,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/test_id2', - to_vertex_id='hive://gold.test_schema1/test_table1/test_id2/_description', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id2/_description', + label='DESCRIPTION' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id2/_description', label='DESCRIPTION' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/test_id2', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/test_id2/_description', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/test_id2', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:hive://gold.test_schema1/test_table1/test_id2/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/test_id2/_description', - to_vertex_id='hive://gold.test_schema1/test_table1/test_id2', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id2/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', + label='DESCRIPTION_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/test_id2/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/test_id2', label='DESCRIPTION_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/test_id2/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/test_id2', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:hive://gold.test_schema1/test_table1/test_id2/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/test_id2', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -295,25 +370,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1/test_table1/is_active', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/is_active', label='COLUMN' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/is_active', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/is_active', + label='COLUMN' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/is_active', NEPTUNE_HEADER_LABEL: 'COLUMN', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/is_active', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/is_active', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='COLUMN_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/is_active', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='COLUMN_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/is_active', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/is_active', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'COLUMN_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -321,25 +406,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1/test_table1/source', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/source', + label='COLUMN' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/source', label='COLUMN' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/source', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/source', NEPTUNE_HEADER_LABEL: 'COLUMN', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/source', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/source', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='COLUMN_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/source', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/source', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='COLUMN_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/source', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'COLUMN_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -347,25 +442,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/source', - to_vertex_id='hive://gold.test_schema1/test_table1/source/_description', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/source', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/source/_description', + label='DESCRIPTION' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/source', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/source/_description', label='DESCRIPTION' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/source', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/source/_description', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/source', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Description:hive://gold.test_schema1/test_table1/source/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/source/_description', - to_vertex_id='hive://gold.test_schema1/test_table1/source', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/source/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/source', + label='DESCRIPTION_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/source/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/source', label='DESCRIPTION_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/source/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/source', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Description:hive://gold.test_schema1/test_table1/source/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/source', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -373,25 +478,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1/test_table1/etl_created_at', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', label='COLUMN' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/etl_created_at', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', + label='COLUMN' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/etl_created_at', NEPTUNE_HEADER_LABEL: 'COLUMN', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/etl_created_at', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='COLUMN_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='COLUMN_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/etl_created_at', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/etl_created_at', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'COLUMN_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -399,25 +514,37 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/etl_created_at', - to_vertex_id='hive://gold.test_schema1/test_table1/etl_created_at/_description', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', + label='DESCRIPTION' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', + to_vertex_id='Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', label='DESCRIPTION' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/etl_created_at', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/etl_created_at/_description', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/etl_created_at', + NEPTUNE_RELATIONSHIP_HEADER_TO: + 'Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', NEPTUNE_HEADER_LABEL: 'DESCRIPTION', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/etl_created_at/_description', - to_vertex_id='hive://gold.test_schema1/test_table1/etl_created_at', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', label='DESCRIPTION_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/etl_created_at/_description', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/etl_created_at', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/etl_created_at', + label='DESCRIPTION_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: + 'Description:hive://gold.test_schema1/test_table1/etl_created_at/_description', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/etl_created_at', NEPTUNE_HEADER_LABEL: 'DESCRIPTION_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -425,25 +552,35 @@ ], [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1', - to_vertex_id='hive://gold.test_schema1/test_table1/ds', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/ds', + label='COLUMN' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.test_schema1/test_table1', + to_vertex_id='Column:hive://gold.test_schema1/test_table1/ds', label='COLUMN' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1/ds', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.test_schema1/test_table1/ds', NEPTUNE_HEADER_LABEL: 'COLUMN', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.test_schema1/test_table1/ds', - to_vertex_id='hive://gold.test_schema1/test_table1', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/ds', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', + label='COLUMN_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.test_schema1/test_table1/ds', + to_vertex_id='Table:hive://gold.test_schema1/test_table1', label='COLUMN_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1/test_table1/ds', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1/test_table1', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.test_schema1/test_table1/ds', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.test_schema1/test_table1', NEPTUNE_HEADER_LABEL: 'COLUMN_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB @@ -453,36 +590,40 @@ { NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'database://hive', - NEPTUNE_HEADER_ID: 'database://hive_hive://gold_CLUSTER', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Database:database://hive', + NEPTUNE_HEADER_ID: 'CLUSTER:Database:database://hive_Cluster:hive://gold', + METADATA_KEY_PROPERTY_NAME: 'CLUSTER:Database:database://hive_Cluster:hive://gold', NEPTUNE_HEADER_LABEL: 'CLUSTER', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold' + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Cluster:hive://gold' }, { NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold', - NEPTUNE_HEADER_ID: 'hive://gold_database://hive_CLUSTER_OF', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Cluster:hive://gold', + NEPTUNE_HEADER_ID: 'CLUSTER_OF:Cluster:hive://gold_Database:database://hive', + METADATA_KEY_PROPERTY_NAME: 'CLUSTER_OF:Cluster:hive://gold_Database:database://hive', NEPTUNE_HEADER_LABEL: 'CLUSTER_OF', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'database://hive' + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Database:database://hive' } ], [ { NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold', - NEPTUNE_HEADER_ID: 'hive://gold_hive://gold.test_schema1_SCHEMA', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Cluster:hive://gold', + NEPTUNE_HEADER_ID: 'SCHEMA:Cluster:hive://gold_Schema:hive://gold.test_schema1', + METADATA_KEY_PROPERTY_NAME: 'SCHEMA:Cluster:hive://gold_Schema:hive://gold.test_schema1', NEPTUNE_HEADER_LABEL: 'SCHEMA', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.test_schema1' + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Schema:hive://gold.test_schema1' }, { NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.test_schema1', - NEPTUNE_HEADER_ID: 'hive://gold.test_schema1_hive://gold_SCHEMA_OF', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Schema:hive://gold.test_schema1', + NEPTUNE_HEADER_ID: 'SCHEMA_OF:Schema:hive://gold.test_schema1_Cluster:hive://gold', + METADATA_KEY_PROPERTY_NAME: 'SCHEMA_OF:Schema:hive://gold.test_schema1_Cluster:hive://gold', NEPTUNE_HEADER_LABEL: 'SCHEMA_OF', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold' + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Cluster:hive://gold' } ] ] diff --git a/tests/unit/models/test_table_column_usage.py b/tests/unit/models/test_table_column_usage.py index d87a68ba1..28f0aec67 100644 --- a/tests/unit/models/test_table_column_usage.py +++ b/tests/unit/models/test_table_column_usage.py @@ -8,9 +8,10 @@ from databuilder.models.table_column_usage import ColumnReader, TableColumnUsage from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, - NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, - NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, + NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, + NEPTUNE_RELATIONSHIP_HEADER_TO, ) @@ -70,26 +71,36 @@ def test_neptune_serialize(self) -> None: rel_row = self.table_col_usage.next_relation() expected = [[ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='db://gold.scm/foo', - to_vertex_id='john@example.com', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:db://gold.scm/foo', + to_vertex_id='User:john@example.com', label='READ_BY' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'db://gold.scm/foo', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'john@example.com', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:db://gold.scm/foo', + to_vertex_id='User:john@example.com', + label='READ_BY' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:db://gold.scm/foo', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'User:john@example.com', NEPTUNE_HEADER_LABEL: 'READ_BY', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'read_count:Long(single)': 1 }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='john@example.com', - to_vertex_id='db://gold.scm/foo', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='User:john@example.com', + to_vertex_id='Table:db://gold.scm/foo', + label='READ' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='User:john@example.com', + to_vertex_id='Table:db://gold.scm/foo', label='READ' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'john@example.com', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'db://gold.scm/foo', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'User:john@example.com', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:db://gold.scm/foo', NEPTUNE_HEADER_LABEL: 'READ', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, diff --git a/tests/unit/models/test_table_last_updated.py b/tests/unit/models/test_table_last_updated.py index 8bb628c53..26cfa9474 100644 --- a/tests/unit/models/test_table_last_updated.py +++ b/tests/unit/models/test_table_last_updated.py @@ -12,7 +12,7 @@ from databuilder.models.timestamp import timestamp_constants from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -51,8 +51,10 @@ def test_create_next_node(self) -> None: self.assertEqual(next_node_serialized, self.expected_node_result) def test_create_next_node_neptune(self) -> None: + node_id = TableLastUpdated.LAST_UPDATED_NODE_LABEL + ":" + self.tableLastUpdated.get_last_updated_model_key() expected_node = { - NEPTUNE_HEADER_ID: self.tableLastUpdated.get_last_updated_model_key(), + NEPTUNE_HEADER_ID: node_id, + METADATA_KEY_PROPERTY_NAME: node_id, NEPTUNE_HEADER_LABEL: TableLastUpdated.LAST_UPDATED_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -75,25 +77,35 @@ def test_create_next_relation_neptune(self) -> None: next_relation_serialized = neptune_serializer.convert_relationship(next_relation) expected = [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.default/test_table', - to_vertex_id='hive://gold.default/test_table/timestamp', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.default/test_table', + to_vertex_id='Timestamp:hive://gold.default/test_table/timestamp', label='LAST_UPDATED_AT' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.default/test_table', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.default/test_table/timestamp', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:hive://gold.default/test_table', + to_vertex_id='Timestamp:hive://gold.default/test_table/timestamp', + label='LAST_UPDATED_AT' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:hive://gold.default/test_table', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Timestamp:hive://gold.default/test_table/timestamp', NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_AT', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.default/test_table/timestamp', - to_vertex_id='hive://gold.default/test_table', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Timestamp:hive://gold.default/test_table/timestamp', + to_vertex_id='Table:hive://gold.default/test_table', + label='LAST_UPDATED_TIME_OF' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Timestamp:hive://gold.default/test_table/timestamp', + to_vertex_id='Table:hive://gold.default/test_table', label='LAST_UPDATED_TIME_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.default/test_table/timestamp', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.default/test_table', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Timestamp:hive://gold.default/test_table/timestamp', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:hive://gold.default/test_table', NEPTUNE_HEADER_LABEL: 'LAST_UPDATED_TIME_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/test_table_lineage.py b/tests/unit/models/test_table_lineage.py index 310360320..2db963046 100644 --- a/tests/unit/models/test_table_lineage.py +++ b/tests/unit/models/test_table_lineage.py @@ -11,9 +11,10 @@ from databuilder.models.table_lineage import TableLineage from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, - NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, - NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, + NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, + NEPTUNE_RELATIONSHIP_HEADER_TO, ) DB = 'hive' @@ -71,25 +72,35 @@ def test_create_relation_neptune(self) -> None: expected = [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.start_key, - to_vertex_id=self.end_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:' + self.start_key, + to_vertex_id='Table:' + self.end_key, label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.start_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.end_key, + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:' + self.start_key, + to_vertex_id='Table:' + self.end_key, + label=TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:' + self.start_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:' + self.end_key, NEPTUNE_HEADER_LABEL: TableLineage.ORIGIN_DEPENDENCY_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.end_key, - to_vertex_id=self.start_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:' + self.end_key, + to_vertex_id='Table:' + self.start_key, + label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Table:' + self.end_key, + to_vertex_id='Table:' + self.start_key, label=TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.end_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.start_key, + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Table:' + self.end_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Table:' + self.start_key, NEPTUNE_HEADER_LABEL: TableLineage.DEPENDENCY_ORIGIN_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/test_table_metadata.py b/tests/unit/models/test_table_metadata.py index 4691ea114..5e16ecc29 100644 --- a/tests/unit/models/test_table_metadata.py +++ b/tests/unit/models/test_table_metadata.py @@ -55,28 +55,28 @@ def test_serialize(self) -> None: 'is_view:UNQUOTED': False}, {'description': 'test_table1', 'KEY': 'hive://gold.test_schema1/test_table1/_description', 'LABEL': 'Description', 'description_source': 'description'}, - {'sort_order:UNQUOTED': 0, 'type': 'bigint', 'name': 'test_id1', + {'sort_order:UNQUOTED': 0, 'col_type': 'bigint', 'name': 'test_id1', 'KEY': 'hive://gold.test_schema1/test_table1/test_id1', 'LABEL': 'Column'}, {'description': 'description of test_table1', 'KEY': 'hive://gold.test_schema1/test_table1/test_id1/_description', 'LABEL': 'Description', 'description_source': 'description'}, - {'sort_order:UNQUOTED': 1, 'type': 'bigint', 'name': 'test_id2', + {'sort_order:UNQUOTED': 1, 'col_type': 'bigint', 'name': 'test_id2', 'KEY': 'hive://gold.test_schema1/test_table1/test_id2', 'LABEL': 'Column'}, {'description': 'description of test_id2', 'KEY': 'hive://gold.test_schema1/test_table1/test_id2/_description', 'LABEL': 'Description', 'description_source': 'description'}, - {'sort_order:UNQUOTED': 2, 'type': 'boolean', 'name': 'is_active', + {'sort_order:UNQUOTED': 2, 'col_type': 'boolean', 'name': 'is_active', 'KEY': 'hive://gold.test_schema1/test_table1/is_active', 'LABEL': 'Column'}, - {'sort_order:UNQUOTED': 3, 'type': 'varchar', 'name': 'source', + {'sort_order:UNQUOTED': 3, 'col_type': 'varchar', 'name': 'source', 'KEY': 'hive://gold.test_schema1/test_table1/source', 'LABEL': 'Column'}, {'description': 'description of source', 'KEY': 'hive://gold.test_schema1/test_table1/source/_description', 'LABEL': 'Description', 'description_source': 'description'}, - {'sort_order:UNQUOTED': 4, 'type': 'timestamp', 'name': 'etl_created_at', + {'sort_order:UNQUOTED': 4, 'col_type': 'timestamp', 'name': 'etl_created_at', 'KEY': 'hive://gold.test_schema1/test_table1/etl_created_at', 'LABEL': 'Column'}, {'description': 'description of etl_created_at', 'KEY': 'hive://gold.test_schema1/test_table1/etl_created_at/_description', 'LABEL': 'Description', 'description_source': 'description'}, - {'sort_order:UNQUOTED': 5, 'type': 'varchar', 'name': 'ds', + {'sort_order:UNQUOTED': 5, 'col_type': 'varchar', 'name': 'ds', 'KEY': 'hive://gold.test_schema1/test_table1/ds', 'LABEL': 'Column'} ] @@ -180,7 +180,7 @@ def test_serialize_neptune(self) -> None: relation_row_serialized = neptune_serializer.convert_relationship(relation_row) neptune_actual.append(relation_row_serialized) relation_row = self.table_metadata.next_relation() - + self.maxDiff = None self.assertEqual(EXPECTED_RELATIONSHIPS_NEPTUNE, neptune_actual) def test_table_attributes(self) -> None: diff --git a/tests/unit/models/test_table_owner.py b/tests/unit/models/test_table_owner.py index 64195591c..e9858c690 100644 --- a/tests/unit/models/test_table_owner.py +++ b/tests/unit/models/test_table_owner.py @@ -12,7 +12,7 @@ from databuilder.models.user import User from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -70,14 +70,16 @@ def test_create_nodes_neptune(self) -> None: nodes = self.table_owner.create_nodes() expected_node1 = { - NEPTUNE_HEADER_ID: User.USER_NODE_KEY_FORMAT.format(email=owner1), + NEPTUNE_HEADER_ID: "User:" + User.USER_NODE_KEY_FORMAT.format(email=owner1), + METADATA_KEY_PROPERTY_NAME: "User:" + User.USER_NODE_KEY_FORMAT.format(email=owner1), NEPTUNE_HEADER_LABEL: User.USER_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, User.USER_NODE_EMAIL + ":String(single)": owner1 } expected_node2 = { - NEPTUNE_HEADER_ID: User.USER_NODE_KEY_FORMAT.format(email=owner2), + NEPTUNE_HEADER_ID: "User:" + User.USER_NODE_KEY_FORMAT.format(email=owner2), + METADATA_KEY_PROPERTY_NAME: "User:" + User.USER_NODE_KEY_FORMAT.format(email=owner2), NEPTUNE_HEADER_LABEL: User.USER_NODE_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -125,25 +127,35 @@ def test_create_relation_neptune(self) -> None: relations = self.table_owner.create_relation() expected = [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=owner1, - to_vertex_id=self.table_owner.get_metadata_model_key(), + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="User:" + owner1, + to_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), label=TableOwner.OWNER_TABLE_RELATION_TYPE ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: owner1, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.table_owner.get_metadata_model_key(), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="User:" + owner1, + to_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), + label=TableOwner.OWNER_TABLE_RELATION_TYPE + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: "User:" + owner1, + NEPTUNE_RELATIONSHIP_HEADER_TO: "Table:" + self.table_owner.get_metadata_model_key(), NEPTUNE_HEADER_LABEL: TableOwner.OWNER_TABLE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.table_owner.get_metadata_model_key(), - to_vertex_id=owner1, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), + to_vertex_id="User:" + owner1, + label=TableOwner.TABLE_OWNER_RELATION_TYPE + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Table:" + self.table_owner.get_metadata_model_key(), + to_vertex_id="User:" + owner1, label=TableOwner.TABLE_OWNER_RELATION_TYPE ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.table_owner.get_metadata_model_key(), - NEPTUNE_RELATIONSHIP_HEADER_TO: owner1, + NEPTUNE_RELATIONSHIP_HEADER_FROM: "Table:" + self.table_owner.get_metadata_model_key(), + NEPTUNE_RELATIONSHIP_HEADER_TO: "User:" + owner1, NEPTUNE_HEADER_LABEL: TableOwner.TABLE_OWNER_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/test_table_source.py b/tests/unit/models/test_table_source.py index 79068d0ac..059ed283b 100644 --- a/tests/unit/models/test_table_source.py +++ b/tests/unit/models/test_table_source.py @@ -11,9 +11,10 @@ from databuilder.models.table_source import TableSource from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, - NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, - NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, + NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, + NEPTUNE_RELATIONSHIP_HEADER_TO, ) DB = 'hive' @@ -70,25 +71,35 @@ def test_create_relation_neptune(self) -> None: expected = [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.start_key, - to_vertex_id=self.end_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Source:" + self.start_key, + to_vertex_id="Table:" + self.end_key, label=TableSource.SOURCE_TABLE_RELATION_TYPE ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.start_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.end_key, + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Source:" + self.start_key, + to_vertex_id="Table:" + self.end_key, + label=TableSource.SOURCE_TABLE_RELATION_TYPE + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: "Source:" + self.start_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: "Table:" + self.end_key, NEPTUNE_HEADER_LABEL: TableSource.SOURCE_TABLE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.end_key, - to_vertex_id=self.start_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Table:" + self.end_key, + to_vertex_id="Source:" + self.start_key, + label=TableSource.TABLE_SOURCE_RELATION_TYPE + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Table:" + self.end_key, + to_vertex_id="Source:" + self.start_key, label=TableSource.TABLE_SOURCE_RELATION_TYPE ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.end_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.start_key, + NEPTUNE_RELATIONSHIP_HEADER_FROM: "Table:" + self.end_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: "Source:" + self.start_key, NEPTUNE_HEADER_LABEL: TableSource.TABLE_SOURCE_RELATION_TYPE, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/test_table_stats.py b/tests/unit/models/test_table_stats.py index 0126ba96b..9ce9d5420 100644 --- a/tests/unit/models/test_table_stats.py +++ b/tests/unit/models/test_table_stats.py @@ -11,7 +11,7 @@ from databuilder.models.table_stats import TableColumnStats from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -33,7 +33,7 @@ def setUp(self) -> None: NODE_KEY: 'hive://gold.base/test/col/avg/', NODE_LABEL: 'Stat', 'stat_val': '1', - 'stat_name': 'avg', + 'stat_type': 'avg', 'start_epoch': '1', 'end_epoch': '2', } @@ -77,12 +77,13 @@ def test_create_next_node_neptune(self) -> None: next_node = self.table_stats.create_next_node() serialized_node = neptune_serializer.convert_node(next_node) expected_neptune_node = { - NEPTUNE_HEADER_ID: 'hive://gold.base/test/col/avg/', + NEPTUNE_HEADER_ID: 'Stat:hive://gold.base/test/col/avg/', + METADATA_KEY_PROPERTY_NAME: 'Stat:hive://gold.base/test/col/avg/', NEPTUNE_HEADER_LABEL: 'Stat', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, 'stat_val:String(single)': '1', - 'stat_name:String(single)': 'avg', + 'stat_type:String(single)': 'avg', 'start_epoch:String(single)': '1', 'end_epoch:String(single)': '2', } @@ -107,25 +108,35 @@ def test_create_next_relation_neptune(self) -> None: expected = [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.base/test/col/avg/', - to_vertex_id='hive://gold.base/test/col', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Stat:hive://gold.base/test/col/avg/', + to_vertex_id='Column:hive://gold.base/test/col', label='STAT_OF' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.base/test/col/avg/', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.base/test/col', + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Stat:hive://gold.base/test/col/avg/', + to_vertex_id='Column:hive://gold.base/test/col', + label='STAT_OF' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Stat:hive://gold.base/test/col/avg/', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Column:hive://gold.base/test/col', NEPTUNE_HEADER_LABEL: 'STAT_OF', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id='hive://gold.base/test/col', - to_vertex_id='hive://gold.base/test/col/avg/', + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.base/test/col', + to_vertex_id='Stat:hive://gold.base/test/col/avg/', + label='STAT' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id='Column:hive://gold.base/test/col', + to_vertex_id='Stat:hive://gold.base/test/col/avg/', label='STAT' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: 'hive://gold.base/test/col', - NEPTUNE_RELATIONSHIP_HEADER_TO: 'hive://gold.base/test/col/avg/', + NEPTUNE_RELATIONSHIP_HEADER_FROM: 'Column:hive://gold.base/test/col', + NEPTUNE_RELATIONSHIP_HEADER_TO: 'Stat:hive://gold.base/test/col/avg/', NEPTUNE_HEADER_LABEL: 'STAT', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/models/test_user.py b/tests/unit/models/test_user.py index 79099c3b1..9a28001de 100644 --- a/tests/unit/models/test_user.py +++ b/tests/unit/models/test_user.py @@ -11,9 +11,10 @@ from databuilder.models.user import User from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, - NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, - NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, + NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, + NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, + NEPTUNE_RELATIONSHIP_HEADER_TO, ) @@ -105,12 +106,17 @@ def test_create_relation_neptune(self) -> None: serialized = neptune_serializer.convert_relationship(relations[0]) - start_key = '{email}'.format(email='test@email.com') - end_key = '{email}'.format(email='test_manager@email.com') + start_key = 'User:{email}'.format(email='test@email.com') + end_key = 'User:{email}'.format(email='test_manager@email.com') expected = [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=start_key, + to_vertex_id=end_key, + label=User.USER_MANAGER_RELATION_TYPE + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=start_key, to_vertex_id=end_key, label=User.USER_MANAGER_RELATION_TYPE @@ -122,7 +128,12 @@ def test_create_relation_neptune(self) -> None: NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id=end_key, + to_vertex_id=start_key, + label=User.MANAGER_USER_RELATION_TYPE + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( from_vertex_id=end_key, to_vertex_id=start_key, label=User.MANAGER_USER_RELATION_TYPE diff --git a/tests/unit/models/test_watermark.py b/tests/unit/models/test_watermark.py index 51d4313f4..a0c467d1b 100644 --- a/tests/unit/models/test_watermark.py +++ b/tests/unit/models/test_watermark.py @@ -13,7 +13,7 @@ from databuilder.models.watermark import Watermark from databuilder.serializers import neo4_serializer, neptune_serializer from databuilder.serializers.neptune_serializer import ( - NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, + METADATA_KEY_PROPERTY_NAME, NEPTUNE_CREATION_TYPE_JOB, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_HEADER_ID, NEPTUNE_HEADER_LABEL, NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT, NEPTUNE_RELATIONSHIP_HEADER_FROM, NEPTUNE_RELATIONSHIP_HEADER_TO, @@ -99,7 +99,8 @@ def test_create_nodes_neptune(self) -> None: nodes = self.watermark.create_nodes() expected_serialized_node_result = { - NEPTUNE_HEADER_ID: self.start_key, + NEPTUNE_HEADER_ID: 'Watermark:' + self.start_key, + METADATA_KEY_PROPERTY_NAME: 'Watermark:' + self.start_key, NEPTUNE_HEADER_LABEL: 'Watermark', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_NODE_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB, @@ -122,25 +123,35 @@ def test_create_relation_neptune(self) -> None: serialized_relation = neptune_serializer.convert_relationship(relation[0]) expected = [ { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.start_key, - to_vertex_id=self.end_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Watermark:" + self.start_key, + to_vertex_id="Table:" + self.end_key, label='BELONG_TO_TABLE' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.start_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.end_key, + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Watermark:" + self.start_key, + to_vertex_id="Table:" + self.end_key, + label='BELONG_TO_TABLE' + ), + NEPTUNE_RELATIONSHIP_HEADER_FROM: "Watermark:" + self.start_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: "Table:" + self.end_key, NEPTUNE_HEADER_LABEL: 'BELONG_TO_TABLE', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB }, { - NEPTUNE_HEADER_ID: "{from_vertex_id}_{to_vertex_id}_{label}".format( - from_vertex_id=self.end_key, - to_vertex_id=self.start_key, + NEPTUNE_HEADER_ID: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Table:" + self.end_key, + to_vertex_id="Watermark:" + self.start_key, + label='WATERMARK' + ), + METADATA_KEY_PROPERTY_NAME: "{label}:{from_vertex_id}_{to_vertex_id}".format( + from_vertex_id="Table:" + self.end_key, + to_vertex_id="Watermark:" + self.start_key, label='WATERMARK' ), - NEPTUNE_RELATIONSHIP_HEADER_FROM: self.end_key, - NEPTUNE_RELATIONSHIP_HEADER_TO: self.start_key, + NEPTUNE_RELATIONSHIP_HEADER_FROM: "Table:" + self.end_key, + NEPTUNE_RELATIONSHIP_HEADER_TO: "Watermark:" + self.start_key, NEPTUNE_HEADER_LABEL: 'WATERMARK', NEPTUNE_LAST_EXTRACTED_AT_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: ANY, NEPTUNE_CREATION_TYPE_RELATIONSHIP_PROPERTY_NAME_BULK_LOADER_FORMAT: NEPTUNE_CREATION_TYPE_JOB diff --git a/tests/unit/resources/fs_neptune_csv_loader/nodes/Actor_6.csv b/tests/unit/resources/fs_neptune_csv_loader/nodes/Actor_6.csv index 639258c4b..9527f3967 100644 --- a/tests/unit/resources/fs_neptune_csv_loader/nodes/Actor_6.csv +++ b/tests/unit/resources/fs_neptune_csv_loader/nodes/Actor_6.csv @@ -1,3 +1,3 @@ -"~id","~label","last_extracted_datetime:Date(single)","creation_type:String(single)","name:String(single)","published_tag:String(single)" -"actor://Tom Cruise","Actor","2020-09-01T01:01:00","job","Top Gun","TESTED" -"actor://Meg Ryan","Actor","2020-09-01T01:01:00","job","Top Gun","TESTED" +"~id","~label","last_extracted_datetime:Date(single)","creation_type:String(single)","name:String(single)","published_tag:String(single)","key:String(single)" +"Actor:actor://Tom Cruise","Actor","2020-09-01T01:01:00","job","Top Gun","TESTED","Actor:actor://Tom Cruise" +"Actor:actor://Meg Ryan","Actor","2020-09-01T01:01:00","job","Top Gun","TESTED","Actor:actor://Meg Ryan" diff --git a/tests/unit/resources/fs_neptune_csv_loader/nodes/City_6.csv b/tests/unit/resources/fs_neptune_csv_loader/nodes/City_6.csv index 5cf0d7d5c..8bff1041d 100644 --- a/tests/unit/resources/fs_neptune_csv_loader/nodes/City_6.csv +++ b/tests/unit/resources/fs_neptune_csv_loader/nodes/City_6.csv @@ -1,3 +1,3 @@ -"~id","~label","last_extracted_datetime:Date(single)","creation_type:String(single)","name:String(single)","published_tag:String(single)" -"city://San Diego","City","2020-09-01T01:01:00","job","Top Gun","TESTED" -"city://Oakland","City","2020-09-01T01:01:00","job","Top Gun","TESTED" \ No newline at end of file +"~id","~label","last_extracted_datetime:Date(single)","creation_type:String(single)","name:String(single)","published_tag:String(single)","key:String(single)" +"City:city://San Diego","City","2020-09-01T01:01:00","job","Top Gun","TESTED","City:city://San Diego" +"City:city://Oakland","City","2020-09-01T01:01:00","job","Top Gun","TESTED","City:city://Oakland" diff --git a/tests/unit/resources/fs_neptune_csv_loader/nodes/Movie_6.csv b/tests/unit/resources/fs_neptune_csv_loader/nodes/Movie_6.csv index 865d0232a..54b8b491f 100644 --- a/tests/unit/resources/fs_neptune_csv_loader/nodes/Movie_6.csv +++ b/tests/unit/resources/fs_neptune_csv_loader/nodes/Movie_6.csv @@ -1,2 +1,2 @@ -"~id","~label","last_extracted_datetime:Date(single)","creation_type:String(single)","name:String(single)","published_tag:String(single)" -"movie://Top Gun","Movie","2020-09-01T01:01:00","job","Top Gun","TESTED" +"~id","~label","last_extracted_datetime:Date(single)","creation_type:String(single)","name:String(single)","published_tag:String(single)","key:String(single)" +"Movie:movie://Top Gun","Movie","2020-09-01T01:01:00","job","Top Gun","TESTED","Movie:movie://Top Gun" diff --git a/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_Actor_ACTOR.csv b/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_Actor_ACTOR.csv index c757899e5..77cb35a7b 100644 --- a/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_Actor_ACTOR.csv +++ b/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_Actor_ACTOR.csv @@ -1,5 +1,5 @@ -"~id","~from","~to","~label","last_extracted_datetime:Date(single)","creation_type:String","published_tag:String(single)" -"movie://Top Gun_actor://Tom Cruise_ACTOR","movie://Top Gun","actor://Tom Cruise","ACTOR","2020-09-01T01:01:00","job","TESTED" -"actor://Tom Cruise_movie://Top Gun_ACTED_IN","actor://Tom Cruise","movie://Top Gun","ACTED_IN","2020-09-01T01:01:00","job","TESTED" -"movie://Top Gun_actor://Meg Ryan_ACTOR","movie://Top Gun","actor://Meg Ryan","ACTOR","2020-09-01T01:01:00","job","TESTED" -"actor://Meg Ryan_movie://Top Gun_ACTED_IN","actor://Meg Ryan","movie://Top Gun","ACTED_IN","2020-09-01T01:01:00","job","TESTED" \ No newline at end of file +"~id","~from","~to","~label","last_extracted_datetime:Date(single)","creation_type:String","published_tag:String(single)","key:String(single)" +"ACTOR:Movie:movie://Top Gun_Actor:actor://Tom Cruise","Movie:movie://Top Gun","Actor:actor://Tom Cruise","ACTOR","2020-09-01T01:01:00","job","TESTED","ACTOR:Movie:movie://Top Gun_Actor:actor://Tom Cruise" +"ACTED_IN:Actor:actor://Tom Cruise_Movie:movie://Top Gun","Actor:actor://Tom Cruise","Movie:movie://Top Gun","ACTED_IN","2020-09-01T01:01:00","job","TESTED","ACTED_IN:Actor:actor://Tom Cruise_Movie:movie://Top Gun" +"ACTOR:Movie:movie://Top Gun_Actor:actor://Meg Ryan","Movie:movie://Top Gun","Actor:actor://Meg Ryan","ACTOR","2020-09-01T01:01:00","job","TESTED","ACTOR:Movie:movie://Top Gun_Actor:actor://Meg Ryan" +"ACTED_IN:Actor:actor://Meg Ryan_Movie:movie://Top Gun","Actor:actor://Meg Ryan","Movie:movie://Top Gun","ACTED_IN","2020-09-01T01:01:00","job","TESTED","ACTED_IN:Actor:actor://Meg Ryan_Movie:movie://Top Gun" diff --git a/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_City_FILMED_AT.csv b/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_City_FILMED_AT.csv index 97d675867..7b1bec260 100644 --- a/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_City_FILMED_AT.csv +++ b/tests/unit/resources/fs_neptune_csv_loader/relationships/Movie_City_FILMED_AT.csv @@ -1,5 +1,5 @@ -"~id","~from","~to","~label","last_extracted_datetime:Date(single)","creation_type:String","published_tag:String(single)" -"movie://Top Gun_city://San Diego_FILMED_AT","movie://Top Gun","city://San Diego","FILMED_AT","2020-09-01T01:01:00","job","TESTED" -"city://San Diego_movie://Top Gun_APPEARS_IN","city://San Diego","movie://Top Gun","APPEARS_IN","2020-09-01T01:01:00","job","TESTED" -"movie://Top Gun_city://Oakland_FILMED_AT","movie://Top Gun","city://Oakland","FILMED_AT","2020-09-01T01:01:00","job","TESTED" -"city://Oakland_movie://Top Gun_APPEARS_IN","city://Oakland","movie://Top Gun","APPEARS_IN","2020-09-01T01:01:00","job","TESTED" +"~id","~from","~to","~label","last_extracted_datetime:Date(single)","creation_type:String","published_tag:String(single)","key:String(single)" +"FILMED_AT:Movie:movie://Top Gun_City:city://San Diego","Movie:movie://Top Gun","City:city://San Diego","FILMED_AT","2020-09-01T01:01:00","job","TESTED","FILMED_AT:Movie:movie://Top Gun_City:city://San Diego" +"APPEARS_IN:City:city://San Diego_Movie:movie://Top Gun","City:city://San Diego","Movie:movie://Top Gun","APPEARS_IN","2020-09-01T01:01:00","job","TESTED","APPEARS_IN:City:city://San Diego_Movie:movie://Top Gun" +"FILMED_AT:Movie:movie://Top Gun_City:city://Oakland","Movie:movie://Top Gun","City:city://Oakland","FILMED_AT","2020-09-01T01:01:00","job","TESTED","FILMED_AT:Movie:movie://Top Gun_City:city://Oakland" +"APPEARS_IN:City:city://Oakland_Movie:movie://Top Gun","City:city://Oakland","Movie:movie://Top Gun","APPEARS_IN","2020-09-01T01:01:00","job","TESTED","APPEARS_IN:City:city://Oakland_Movie:movie://Top Gun"