diff --git a/backend/dataall/modules/catalog/api/types.py b/backend/dataall/modules/catalog/api/types.py index f8aa049e0..a33ba37c8 100644 --- a/backend/dataall/modules/catalog/api/types.py +++ b/backend/dataall/modules/catalog/api/types.py @@ -42,21 +42,21 @@ fields=[ gql.Field(name='nodeUri', type=gql.ID), gql.Field(name='parentUri', type=gql.NonNullableType(gql.String)), - gql.Field(name='status', type=gql.NonNullableType(gql.String)), gql.Field(name='owner', type=gql.NonNullableType(gql.String)), gql.Field(name='path', type=gql.NonNullableType(gql.String)), gql.Field(name='label', type=gql.NonNullableType(gql.String)), + gql.Field(name='status', type=gql.NonNullableType(gql.String)), + gql.Field(name='readme', type=gql.String), + gql.Field(name='created', type=gql.NonNullableType(gql.String)), + gql.Field(name='updated', type=gql.String), + gql.Field(name='deleted', type=gql.String), + gql.Field(name='isMatch', type=gql.Boolean), gql.Field(name='admin', type=gql.String), gql.Field( name='userRoleForGlossary', type=GlossaryRole.toGraphQLEnum(), resolver=resolve_user_role, ), - gql.Field(name='readme', type=gql.String), - gql.Field(name='created', type=gql.NonNullableType(gql.String)), - gql.Field(name='updated', type=gql.String), - gql.Field(name='deleted', type=gql.String), - gql.Field(name='isMatch', type=gql.Boolean), gql.Field(name='stats', resolver=resolve_stats, type=gql.Ref('GlossaryNodeStatistics')), gql.Field( resolver=resolve_node_tree, diff --git a/tests_new/integration_tests/modules/catalog/__init__.py b/tests_new/integration_tests/modules/catalog/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests_new/integration_tests/modules/catalog/conftest.py b/tests_new/integration_tests/modules/catalog/conftest.py new file mode 100644 index 000000000..bcbd2f7bb --- /dev/null +++ b/tests_new/integration_tests/modules/catalog/conftest.py @@ -0,0 +1,133 @@ +import pytest +from integration_tests.modules.catalog.queries import ( + create_glossary, + delete_glossary, + create_term, + delete_term, + create_category, + delete_category, + list_glossary_associations, +) + +from integration_tests.modules.s3_datasets.queries import update_dataset + +""" +Temp glossary elements. Scope=function +""" + + +@pytest.fixture(scope='function') +def glossary1(client1, group1): + glos = None + try: + glos = create_glossary( + client1, name='glossary1', group=group1, read_me='Glossary created for integration testing' + ) + yield glos + finally: + if glos: + delete_glossary(client1, node_uri=glos.nodeUri) + + +@pytest.fixture(scope='function') +def category1(client1, group1, glossary1): + cat = None + try: + cat = create_category( + client1, name='category1', parent_uri=glossary1.nodeUri, read_me='Category created for integration testing' + ) + yield cat + finally: + if cat: + delete_category(client1, node_uri=cat.nodeUri) + + +@pytest.fixture(scope='function') +def glossary_term1(client1, group1, glossary1): + term = None + try: + term = create_term( + client1, name='glos_term1', parent_uri=glossary1.nodeUri, read_me='Term created for integration testing' + ) + yield term + finally: + if term: + delete_term(client1, node_uri=term.nodeUri) + + +@pytest.fixture(scope='function') +def category_term1(client1, group1, category1): + term = None + try: + term = create_term( + client1, name='cat_term1', parent_uri=category1.nodeUri, read_me='Term created for integration testing' + ) + yield term + finally: + if term: + delete_term(client1, node_uri=term.nodeUri) + + +""" +Session glossary elements needed if using associations + +WARNING! +Associations are applied to the S3_Datasets module +Glossaries can only be tested if the S3_datasets module is enabled in the deployment used for testing! +""" + + +@pytest.fixture(scope='session') +def session_glossary1(client1, group1): + glos = None + try: + glos = create_glossary( + client1, name='Sesssion glossary1', group=group1, read_me='Glossary created for integration testing' + ) + yield glos + finally: + if glos: + delete_glossary(client1, node_uri=glos.nodeUri) + + +@pytest.fixture(scope='session') +def session_glossary_term1(client1, group1, session_glossary1): + term = None + try: + term = create_term( + client1, + name='Session glos_term1', + parent_uri=session_glossary1.nodeUri, + read_me='Term created for integration testing', + ) + yield term + finally: + if term: + delete_term(client1, node_uri=term.nodeUri) + + +@pytest.fixture(scope='session') +def dataset_association_with_glossary_term1( + client1, group1, session_glossary1, session_glossary_term1, session_s3_dataset1 +): + update_dataset( + client1, + datasetUri=session_s3_dataset1.datasetUri, + input={ + 'terms': [session_glossary_term1.nodeUri], + 'label': session_s3_dataset1.label, + 'description': session_s3_dataset1.description, + 'tags': session_s3_dataset1.tags, + 'stewards': session_s3_dataset1.stewards, + 'topics': session_s3_dataset1.topics, + 'confidentiality': session_s3_dataset1.confidentiality, + 'autoApprovalEnabled': False, + 'enableExpiration': False, + 'KmsAlias': session_s3_dataset1.KmsAlias, + }, + ) + response = list_glossary_associations(client1, node_uri=session_glossary1.nodeUri) + ds_association = next( + (assoc for assoc in response.associations.nodes if assoc.targetUri == session_s3_dataset1.datasetUri), None + ) + yield ds_association diff --git a/tests_new/integration_tests/modules/catalog/queries.py b/tests_new/integration_tests/modules/catalog/queries.py new file mode 100644 index 000000000..e19f5395a --- /dev/null +++ b/tests_new/integration_tests/modules/catalog/queries.py @@ -0,0 +1,421 @@ +# TODO: This file will be replaced by using the SDK directly + +GLOSSARY_TERM_CATEGORY_COMMON_FIELDS = """ +nodeUri, +parentUri, +owner, +path, +label, +status, +readme, +created, +updated, +deleted, +isMatch +""" + + +def create_glossary(client, name, group, read_me): + query = { + 'operationName': 'CreateGlossary', + 'variables': { + 'input': { + 'label': name, + 'admin': group, + 'status': 'approved', + 'readme': read_me, + } + }, + 'query': f""" + mutation CreateGlossary($input: CreateGlossaryInput) {{ + createGlossary(input: $input) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + """, + } + response = client.query(query=query) + return response.data.createGlossary + + +def update_glossary(client, node_uri, name, group, read_me): + query = { + 'operationName': 'UpdateGlossary', + 'variables': { + 'nodeUri': node_uri, + 'input': { + 'label': name, + 'admin': group, + 'status': 'SomeStatus', + 'readme': read_me, + }, + }, + 'query': f""" + mutation UpdateGlossary($nodeUri: String!, $input: UpdateGlossaryInput) {{ + updateGlossary(nodeUri: $nodeUri, input: $input) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + """, + } + response = client.query(query=query) + return response.data.updateGlossary + + +def delete_glossary(client, node_uri): + query = { + 'operationName': 'deleteGlossary', + 'variables': {'nodeUri': node_uri}, + 'query': """ + mutation deleteGlossary($nodeUri: String!) { + deleteGlossary(nodeUri: $nodeUri) + } + """, + } + response = client.query(query=query) + return response.data.deleteGlossary + + +def get_glossary(client, node_uri): + query = { + 'operationName': 'GetGlossary', + 'variables': {'nodeUri': node_uri}, + 'query': f""" + query GetGlossary($nodeUri: String!) {{ + getGlossary(nodeUri: $nodeUri) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + admin + userRoleForGlossary + stats {{ + categories + terms + associations + }} + }} + }} + """, + } + response = client.query(query=query) + return response.data.getGlossary + + +def get_glossary_tree(client, node_uri, node_type=''): + query = { + 'operationName': 'GetGlossaryTree', + 'variables': {'nodeUri': node_uri, 'filter': {'nodeType': node_type}}, + 'query': f""" + query GetGlossaryTree( + $nodeUri: String! + $filter: GlossaryNodeSearchFilter + ) {{ + getGlossary(nodeUri: $nodeUri) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + admin + categories {{ + count + page + pages + hasNext + hasPrevious + nodes {{ + parentUri + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + stats {{ + categories + terms + }} + }} + }} + tree(filter: $filter) {{ + count + hasNext + hasPrevious + page + pages + nodes {{ + __typename + ... on Glossary {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + ... on Category {{ + parentUri + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + ... on Term {{ + parentUri + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + }} + }} + }} + """, + } + response = client.query(query=query) + return response.data.getGlossary + + +def list_glossary_associations(client, node_uri): + query = { + 'operationName': 'GetGlossaryTree', + 'variables': {'nodeUri': node_uri}, + 'query': f""" + query GetGlossaryTree( + $nodeUri: String! + $filter: GlossaryTermTargetFilter + ) {{ + getGlossary(nodeUri: $nodeUri) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + admin + userRoleForGlossary + associations(filter: $filter) {{ + count + page + pages + hasNext + hasPrevious + nodes {{ + linkUri + targetUri + approvedBySteward + term {{ + label + nodeUri + }} + targetType + target {{ + label + }} + }} + }} + }} + }} + """, + } + response = client.query(query=query) + return response.data.getGlossary + + +def list_glossaries(client, term='', status=''): + query = { + 'operationName': 'ListGlossaries', + 'variables': {'filter': {'term': term, 'status': status}}, + 'query': f""" + query ListGlossaries($filter: GlossaryFilter) {{ + listGlossaries(filter: $filter) {{ + count + page + pages + hasNext + hasPrevious + nodes {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + admin + stats {{ + categories + terms + associations + }} + }} + }} + }} + """, + } + response = client.query(query=query) + return response.data.listGlossaries + + +def search_glossary(client, term='', node_type=''): + query = { + 'operationName': 'SearchGlossary', + 'variables': {'filter': {'term': term, 'nodeType': node_type}}, + 'query': f""" + query SearchGlossary($filter: GlossaryNodeSearchFilter) {{ + searchGlossary(filter: $filter) {{ + count + page + pages + hasNext + hasPrevious + nodes {{ + __typename + ... on Glossary {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + ... on Category {{ + parentUri + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + ... on Term {{ + parentUri + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + }} + }} + """, + } + response = client.query(query=query) + return response.data.searchGlossary + + +def create_category(client, parent_uri, name, read_me): + query = { + 'operationName': 'CreateCategory', + 'variables': { + 'parentUri': parent_uri, + 'input': { + 'label': name, + 'status': 'approved', + 'readme': read_me, + }, + }, + 'query': f""" + mutation CreateCategory($parentUri: String!, $input: CreateCategoryInput) {{ + createCategory(parentUri: $parentUri, input: $input) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + """, + } + response = client.query(query=query) + return response.data.createCategory + + +def update_category(client, node_uri, name, read_me): + query = { + 'operationName': 'UpdateCategory', + 'variables': { + 'nodeUri': node_uri, + 'input': { + 'label': name, + 'status': 'SomeStatus', + 'readme': read_me, + }, + }, + 'query': f""" + mutation UpdateCategory($nodeUri: String!, $input: UpdateCategoryInput) {{ + updateCategory(nodeUri: $nodeUri, input: $input) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + """, + } + response = client.query(query=query) + return response.data.updateCategory + + +def delete_category(client, node_uri): + query = { + 'operationName': 'deleteCategory', + 'variables': {'nodeUri': node_uri}, + 'query': """ + mutation deleteCategory($nodeUri: String!) { + deleteCategory(nodeUri: $nodeUri) + } + """, + } + response = client.query(query=query) + return response.data.deleteCategory + + +def create_term(client, parent_uri, name, read_me): + query = { + 'operationName': 'CreateTerm', + 'variables': { + 'parentUri': parent_uri, + 'input': { + 'label': name, + 'status': 'SomeStatus', + 'readme': read_me, + }, + }, + 'query': f""" + mutation CreateTerm($parentUri: String!, $input: CreateTermInput) {{ + createTerm(parentUri: $parentUri, input: $input) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + """, + } + response = client.query(query=query) + return response.data.createTerm + + +def update_term(client, node_uri, name, read_me): + query = { + 'operationName': 'UpdateTerm', + 'variables': { + 'nodeUri': node_uri, + 'input': { + 'label': name, + 'status': 'SomeStatus', + 'readme': read_me, + }, + }, + 'query': f""" + mutation UpdateTerm($nodeUri: String!, $input: UpdateTermInput) {{ + updateTerm(nodeUri: $nodeUri, input: $input) {{ + {GLOSSARY_TERM_CATEGORY_COMMON_FIELDS} + }} + }} + """, + } + response = client.query(query=query) + return response.data.updateTerm + + +def delete_term(client, node_uri): + query = { + 'operationName': 'deleteTerm', + 'variables': {'nodeUri': node_uri}, + 'query': """ + mutation deleteTerm($nodeUri: String!) { + deleteTerm(nodeUri: $nodeUri) + } + """, + } + response = client.query(query=query) + return response.data.deleteTerm + + +def approve_term_association(client, link_uri): + query = { + 'operationName': 'ApproveTermAssociation', + 'variables': {'linkUri': link_uri}, + 'query': """ + mutation ApproveTermAssociation($linkUri: String!) { + approveTermAssociation(linkUri: $linkUri) + } + """, + } + response = client.query(query=query) + return response.data.approveTermAssociation + + +def dismiss_term_association(client, link_uri): + query = { + 'operationName': 'DismissTermAssociation', + 'variables': {'linkUri': link_uri}, + 'query': """ + mutation DismissTermAssociation($linkUri: String!) { + dismissTermAssociation(linkUri: $linkUri) + } + """, + } + response = client.query(query=query) + return response.data.dismissTermAssociation + + +def start_reindex_catalog(client, handle_deletes): + query = { + 'operationName': 'startReindexCatalog', + 'variables': {'handleDeletes': handle_deletes}, + 'query': """ + mutation startReindexCatalog($handleDeletes: Boolean!) { + startReindexCatalog(handleDeletes: $handleDeletes) + } + """, + } + response = client.query(query=query) + return response.data.startReindexCatalog diff --git a/tests_new/integration_tests/modules/catalog/test_glossaries.py b/tests_new/integration_tests/modules/catalog/test_glossaries.py new file mode 100644 index 000000000..b7636bb74 --- /dev/null +++ b/tests_new/integration_tests/modules/catalog/test_glossaries.py @@ -0,0 +1,216 @@ +from assertpy import assert_that + +from integration_tests.errors import GqlError +from integration_tests.modules.catalog.queries import ( + create_glossary, + get_glossary, + get_glossary_tree, + list_glossary_associations, + list_glossaries, + search_glossary, + delete_glossary, + update_glossary, + create_term, + delete_term, + update_term, + create_category, + delete_category, + update_category, + approve_term_association, + dismiss_term_association, + start_reindex_catalog, +) + + +def test_create_glossary(client1, glossary1): + assert_that(glossary1.nodeUri).is_not_none() + assert_that(glossary1.label).is_equal_to('glossary1') + assert_that(glossary1.readme).is_equal_to('Glossary created for integration testing') + + +def test_get_glossary(client1, glossary1, category1, glossary_term1, category_term1): + response = get_glossary(client1, node_uri=glossary1.nodeUri) + assert_that(response.label).is_equal_to('glossary1') + assert_that(response.nodeUri).is_equal_to(glossary1.nodeUri) + assert_that(response.stats).contains_entry(categories=1, terms=2) + + +def test_get_glossary_get_tree(client1, glossary1, category1, glossary_term1, category_term1): + response = get_glossary_tree(client1, node_uri=glossary1.nodeUri) + assert_that(response.tree).is_not_none() + assert_that(response.tree.count).is_equal_to(4) + # Check that the glossary is in the tree + glos = next((n for n in response.tree.nodes if n.nodeUri == glossary1.nodeUri), None) + assert_that(glos.label).is_equal_to(glossary1.label) + assert_that(glos.nodeUri).is_equal_to(glossary1.nodeUri) + assert_that(glos.path).is_equal_to(f'/{glossary1.nodeUri}') + # Check that the category is in the tree + cat = next((n for n in response.tree.nodes if n.nodeUri == category1.nodeUri), None) + assert_that(cat.label).is_equal_to(category1.label) + assert_that(cat.nodeUri).is_equal_to(category1.nodeUri) + assert_that(cat.parentUri).is_equal_to(glossary1.nodeUri) + assert_that(cat.path).is_equal_to(f'/{glossary1.nodeUri}/{category1.nodeUri}') + # Check that the terms are in the tree and their parentUris are correct + g_term = next((n for n in response.tree.nodes if n.nodeUri == glossary_term1.nodeUri), None) + assert_that(g_term.label).is_equal_to(glossary_term1.label) + assert_that(g_term.nodeUri).is_equal_to(glossary_term1.nodeUri) + assert_that(g_term.parentUri).is_equal_to(glossary1.nodeUri) + assert_that(g_term.path).is_equal_to(f'/{glossary1.nodeUri}/{glossary_term1.nodeUri}') + c_term = next((n for n in response.tree.nodes if n.nodeUri == category_term1.nodeUri), None) + assert_that(c_term.label).is_equal_to(category_term1.label) + assert_that(c_term.nodeUri).is_equal_to(category_term1.nodeUri) + assert_that(c_term.parentUri).is_equal_to(category1.nodeUri) + assert_that(c_term.path).is_equal_to(f'/{glossary1.nodeUri}/{category1.nodeUri}/{category_term1.nodeUri}') + + +def test_get_glossary_list_associations( + client1, session_glossary1, session_glossary_term1, dataset_association_with_glossary_term1 +): + response = list_glossary_associations(client1, node_uri=session_glossary1.nodeUri) + assert_that(response.associations.count).is_equal_to(1) + ass = response.associations.nodes[0] + assert_that(ass.linkUri).is_not_none() + assert_that(ass.term.nodeUri).is_equal_to(session_glossary_term1.nodeUri) + + +def test_list_glossaries(client1, glossary1, category1, glossary_term1, category_term1): + response = list_glossaries(client1) + assert_that(response.count).is_greater_than_or_equal_to(1) + glos_1 = next((n for n in response.nodes if n.nodeUri == glossary1.nodeUri), None) + assert_that(glos_1.nodeUri).is_equal_to(glossary1.nodeUri) + assert_that(glos_1.stats).contains_entry(categories=1, terms=2) + + +def test_search_glossary(client1, glossary1, category1, glossary_term1, category_term1): + response = search_glossary(client1, term=glossary1.label) + assert_that(response.count).is_equal_to(1) + + +def test_update_glossary(client1, group1, glossary1, session_id): + response = update_glossary( + client1, + node_uri=glossary1.nodeUri, + name='glossaryUpdated', + group=group1, + read_me=f'UPDATED: {session_id} Glossary created for integration testing', + ) + assert_that(response.label).is_equal_to('glossaryUpdated') + assert_that(response.readme).is_equal_to(f'UPDATED: {session_id} Glossary created for integration testing') + + +def test_delete_glossary(client1, group1): + glos = create_glossary(client1, name='glossary1', group=group1, read_me='Glossary created for integration testing') + response = delete_glossary(client1, glos.nodeUri) + assert_that(response).is_true() + + +def test_delete_glossary_with_categories_and_terms(client1, group1): + glos = create_glossary(client1, name='glossary1', group=group1, read_me='Glossary created for integration testing') + category = create_category( + client1, name='category1', parent_uri=glos.nodeUri, read_me='Category created for integration testing' + ) + term = create_term( + client1, name='term1', parent_uri=category.nodeUri, read_me='Term created for integration testing' + ) + response = delete_glossary(client1, glos.nodeUri) + assert_that(response).is_true() + + +def test_create_category(client1, category1): + assert_that(category1.nodeUri).is_not_none() + assert_that(category1.label).is_equal_to('category1') + + +def test_update_category(client1, category1, session_id): + response = update_category( + client1, + node_uri=category1.nodeUri, + name=category1.label, + read_me=f'UPDATED: {session_id} Category created for integration testing', + ) + assert_that(response.readme).is_equal_to(f'UPDATED: {session_id} Category created for integration testing') + + +def test_delete_category(client1, glossary1): + category = create_category( + client1, name='glossary1', parent_uri=glossary1.nodeUri, read_me='Category created for integration testing' + ) + response = delete_category(client1, category.nodeUri) + assert_that(response).is_true() + + +def test_delete_category_with_terms(client1, glossary1): + category = create_category( + client1, name='category1', parent_uri=glossary1.nodeUri, read_me='Category created for integration testing' + ) + term = create_term( + client1, name='term1', parent_uri=category.nodeUri, read_me='Term created for integration testing' + ) + response = delete_category(client1, node_uri=category.nodeUri) + assert_that(response).is_true() + + +def test_create_term_in_glossary(client1, glossary_term1): + assert_that(glossary_term1.nodeUri).is_not_none() + assert_that(glossary_term1.label).is_equal_to('glos_term1') + + +def test_create_term_in_category(client1, category_term1): + assert_that(category_term1.nodeUri).is_not_none() + assert_that(category_term1.label).is_equal_to('cat_term1') + + +def test_update_term(client1, glossary_term1, session_id): + response = update_term( + client1, + node_uri=glossary_term1.nodeUri, + name=glossary_term1.label, + read_me=f'UPDATED: {session_id} Glossary term created for integration testing', + ) + assert_that(response.readme).is_equal_to(f'UPDATED: {session_id} Glossary term created for integration testing') + + +def test_delete_term(client1, group1, category1): + term = create_term( + client1, name='toDelete', parent_uri=category1.nodeUri, read_me='Term created for integration testing' + ) + response = delete_term(client1, node_uri=term.nodeUri) + assert_that(response).is_true() + + +def test_approve_term_association( + client1, dataset_association_with_glossary_term1, session_glossary1, session_glossary_term1 +): + response = approve_term_association(client1, link_uri=dataset_association_with_glossary_term1.linkUri) + assert_that(response).is_true() + response = list_glossary_associations(client1, node_uri=session_glossary1.nodeUri) + association = next( + (n for n in response.associations.nodes if n.linkUri == dataset_association_with_glossary_term1.linkUri), None + ) + assert_that(association.approvedBySteward).is_equal_to(True) + + +def test_dismiss_term_association(client1, session_glossary1, dataset_association_with_glossary_term1): + response = dismiss_term_association(client1, link_uri=dataset_association_with_glossary_term1.linkUri) + assert_that(response).is_true() + response = list_glossary_associations(client1, node_uri=session_glossary1.nodeUri) + association = next( + (n for n in response.associations.nodes if n.linkUri == dataset_association_with_glossary_term1.linkUri), None + ) + assert_that(association.approvedBySteward).is_equal_to(False) + + +def test_start_reindex_catalog_unauthorized(client1): + assert_that(start_reindex_catalog).raises(GqlError).when_called_with(client1, handle_deletes=True).contains( + 'Only data.all admin', 're-index catalog' + ) + + +def test_start_reindex_catalog_handle_deletes(clientTenant): + response = start_reindex_catalog(clientTenant, handle_deletes=True) + assert_that(response).is_true() + + +def test_start_reindex_catalog_handle_deletes_false(clientTenant): + response = start_reindex_catalog(clientTenant, handle_deletes=False) + assert_that(response).is_true()