diff --git a/backend/dataall/core/environment/cdk/environment_stack.py b/backend/dataall/core/environment/cdk/environment_stack.py index 68d1483b8..2638001a0 100644 --- a/backend/dataall/core/environment/cdk/environment_stack.py +++ b/backend/dataall/core/environment/cdk/environment_stack.py @@ -592,6 +592,8 @@ def create_integration_tests_role(self): 'arn:aws:s3:::dataalltesting*/*', 'arn:aws:s3:::dataall-session*', 'arn:aws:s3:::dataall-session*/*', + 'arn:aws:s3:::dataall-test-session*', + 'arn:aws:s3:::dataall-test-session*/*', 'arn:aws:s3:::dataall-temp*', 'arn:aws:s3:::dataall-temp*/*', ], @@ -613,6 +615,7 @@ def create_integration_tests_role(self): iam.PolicyStatement( actions=[ 'lakeformation:GrantPermissions', + 'lakeformation:RevokePermissions', 'lakeformation:PutDataLakeSettings', 'lakeformation:GetDataLakeSettings', 'glue:GetDatabase', @@ -674,9 +677,14 @@ def create_integration_tests_role(self): 'iam:DeleteRole', 'iam:PutRolePolicy', 'iam:DeleteRolePolicy', + 'iam:DetachRolePolicy', + 'iam:ListAttachedRolePolicies', ], effect=iam.Effect.ALLOW, - resources=[f'arn:aws:iam::{self.account}:role/dataall-test-*'], + resources=[ + f'arn:aws:iam::{self.account}:role/dataall-test-*', + f'arn:aws:iam::{self.account}:role/dataall-session*', + ], ), ) diff --git a/tests_new/clean_up_s3.sh b/tests_new/clean_up_s3.sh deleted file mode 100644 index 39d4c1fe9..000000000 --- a/tests_new/clean_up_s3.sh +++ /dev/null @@ -1,13 +0,0 @@ -my_array=("$(aws s3api list-buckets --query 'Buckets[?contains(Name, `session`) == `true`].[Name]' --output text)") -array=("${(@f)my_array}") -for YOUR_BUCKET in "${array[@]}" -do - -aws s3api delete-objects --bucket ${YOUR_BUCKET} \ ---delete "$(aws s3api list-object-versions --bucket ${YOUR_BUCKET} --query='{Objects: Versions[].{Key:Key,VersionId:VersionId}}')" - -aws s3api delete-objects --bucket ${YOUR_BUCKET} \ ---delete "$(aws s3api list-object-versions --bucket ${YOUR_BUCKET} --query='{Objects: DeleteMarkers[].{Key:Key,VersionId:VersionId}}')" - -aws s3api delete-bucket --bucket ${YOUR_BUCKET} -done \ No newline at end of file diff --git a/tests_new/integration_tests/core/environment/global_conftest.py b/tests_new/integration_tests/core/environment/global_conftest.py index 87b939109..1bf5057ea 100644 --- a/tests_new/integration_tests/core/environment/global_conftest.py +++ b/tests_new/integration_tests/core/environment/global_conftest.py @@ -1,7 +1,8 @@ import logging +from contextlib import contextmanager import pytest -import boto3 +from assertpy import assert_that from integration_tests.aws_clients.sts import STSClient from integration_tests.client import GqlError @@ -14,18 +15,41 @@ ) from integration_tests.core.organizations.queries import create_organization from integration_tests.core.stack.utils import check_stack_ready -from tests_new.integration_tests.core.environment.utils import update_env_stack from tests_new.integration_tests.aws_clients.s3 import S3Client +from tests_new.integration_tests.core.environment.utils import update_env_stack log = logging.getLogger(__name__) -def create_env(client, env_name, group, org_uri, account_id, region, tags=[]): - env = create_environment( - client, name=env_name, group=group, organizationUri=org_uri, awsAccountId=account_id, region=region, tags=tags - ) - check_stack_ready(client, env.environmentUri, env.stack.stackUri) - return get_environment(client, env.environmentUri) +@contextmanager +def create_env(client, env_name, group, org_uri, account_id, region, tags=[], retain=False): + env = None + errors = False + try: + env = create_environment( + client, + name=env_name, + group=group, + organizationUri=org_uri, + awsAccountId=account_id, + region=region, + tags=tags, + ) + check_stack_ready(client, env.environmentUri, env.stack.stackUri) + env = get_environment(client, env.environmentUri) + assert_that(env.stack.status).is_in('CREATE_COMPLETE', 'UPDATE_COMPLETE') + yield env + except Exception as e: + errors = True + raise e + finally: + if env and (not retain or errors): + role = f'arn:aws:iam::{env.AwsAccountId}:role/dataall-integration-tests-role-{env.region}' + session = STSClient(role_arn=role, region=env.region, session_name='Session_1').get_refreshable_session() + S3Client(session=session, account=env.AwsAccountId, region=env.region).delete_bucket( + env.EnvironmentDefaultBucketName + ) + delete_env(client, env) def delete_env(client, env): @@ -46,21 +70,11 @@ def delete_env(client, env): @pytest.fixture(scope='session') def session_env1(client1, group1, group5, org1, session_id, testdata): envdata = testdata.envs['session_env1'] - env = None - try: - env = create_env( - client1, 'session_env1', group1, org1.organizationUri, envdata.accountId, envdata.region, tags=[session_id] - ) + with create_env( + client1, 'session_env1', group1, org1.organizationUri, envdata.accountId, envdata.region, tags=[session_id] + ) as env: invite_group_on_env(client1, env.environmentUri, group5, ['CREATE_DATASET', 'CREATE_SHARE_OBJECT']) yield env - finally: - if env: - role = f'arn:aws:iam::{env.AwsAccountId}:role/dataall-integration-tests-role-{env.region}' - session = STSClient(role_arn=role, region=env.region, session_name='Session_1').get_refreshable_session() - S3Client(session=session, account=env.AwsAccountId, region=env.region).delete_bucket( - env.EnvironmentDefaultBucketName - ) - delete_env(client1, env) @pytest.fixture(scope='session') @@ -78,21 +92,16 @@ def session_env1_aws_client(session_env1, session_env1_integration_role_arn): @pytest.fixture(scope='session') def session_cross_acc_env_1(client5, group5, testdata, org1, session_id): envdata = testdata.envs['session_cross_acc_env_1'] - env = None - try: - env = create_env( - client5, - 'session_cross_acc_env_1', - group5, - org1.organizationUri, - envdata.accountId, - envdata.region, - tags=[session_id], - ) + with create_env( + client5, + 'session_cross_acc_env_1', + group5, + org1.organizationUri, + envdata.accountId, + envdata.region, + tags=[session_id], + ) as env: yield env - finally: - if env: - delete_env(client5, env) @pytest.fixture(scope='session') @@ -124,16 +133,11 @@ def persistent_env1_aws_client(persistent_env1, persistent_env1_integration_role @pytest.fixture(scope='session') def session_env2(client1, group1, group2, org2, session_id, testdata): envdata = testdata.envs['session_env2'] - env = None - try: - env = create_env( - client1, 'session_env2', group1, org2.organizationUri, envdata.accountId, envdata.region, tags=[session_id] - ) + with create_env( + client1, 'session_env2', group1, org2.organizationUri, envdata.accountId, envdata.region, tags=[session_id] + ) as env: invite_group_on_env(client1, env.environmentUri, group2, ['CREATE_DATASET']) yield env - finally: - if env: - delete_env(client1, env) """ @@ -145,13 +149,8 @@ def session_env2(client1, group1, group2, org2, session_id, testdata): @pytest.fixture(scope='function') def temp_env1(client1, group1, org1, testdata): envdata = testdata.envs['temp_env1'] - env = None - try: - env = create_env(client1, 'temp_env1', group1, org1.organizationUri, envdata.accountId, envdata.region) + with create_env(client1, 'temp_env1', group1, org1.organizationUri, envdata.accountId, envdata.region) as env: yield env - finally: - if env: - delete_env(client1, env) """ @@ -160,43 +159,39 @@ def temp_env1(client1, group1, org1, testdata): """ +@contextmanager def get_or_create_persistent_env(env_name, client, group, testdata): envs = list_environments(client, term=env_name).nodes if envs: - return envs[0] + env = envs[0] + update_env_stack(client, env) + yield get_environment(client, env.environmentUri) else: envdata = testdata.envs[env_name] org = create_organization(client, f'org_{env_name}', group) - env = create_env( - client, env_name, group, org.organizationUri, envdata.accountId, envdata.region, tags=[env_name] - ) - if env.stack.status in ['CREATE_COMPLETE', 'UPDATE_COMPLETE']: - return env - else: - delete_env(client, env) - raise RuntimeError(f'failed to create {env_name=} {env=}') + with create_env( + client, + env_name, + group, + org.organizationUri, + envdata.accountId, + envdata.region, + tags=[env_name], + retain=True, + ) as env: + yield env @pytest.fixture(scope='session') def persistent_env1(client1, group1, testdata): - return get_or_create_persistent_env('persistent_env1', client1, group1, testdata) - - -@pytest.fixture(scope='session') -def updated_persistent_env1(client1, group1, persistent_env1): - update_env_stack(client1, persistent_env1) - return get_environment(client1, persistent_env1.environmentUri) + with get_or_create_persistent_env('persistent_env1', client1, group1, testdata) as env: + yield env @pytest.fixture(scope='session') def persistent_cross_acc_env_1(client5, group5, testdata): - return get_or_create_persistent_env('persistent_cross_acc_env_1', client5, group5, testdata) - - -@pytest.fixture(scope='session') -def updated_persistent_cross_acc_env_1(client5, group5, persistent_cross_acc_env_1): - update_env_stack(client5, persistent_cross_acc_env_1) - return get_environment(client5, persistent_cross_acc_env_1.environmentUri) + with get_or_create_persistent_env('persistent_cross_acc_env_1', client5, group5, testdata) as env: + yield env @pytest.fixture(scope='session') diff --git a/tests_new/integration_tests/modules/s3_datasets/aws_clients.py b/tests_new/integration_tests/modules/s3_datasets/aws_clients.py index 4435bb4cf..de01f8357 100644 --- a/tests_new/integration_tests/modules/s3_datasets/aws_clients.py +++ b/tests_new/integration_tests/modules/s3_datasets/aws_clients.py @@ -2,6 +2,8 @@ import json import re import os +from typing import List + from botocore.exceptions import ClientError log = logging.getLogger(__name__) @@ -312,3 +314,12 @@ def grant_create_database(self, role_arn): return True except ClientError as e: log.exception('Error granting permissions to create database') + + def revoke_db_perms(self, principal_arn: str, db_name: str, permissions: List[str]): + self._client.revoke_permissions( + Principal={ + 'DataLakePrincipalIdentifier': principal_arn, + }, + Resource={'Database': {'Name': db_name}}, + Permissions=permissions, + ) diff --git a/tests_new/integration_tests/modules/shares/queries.py b/tests_new/integration_tests/modules/shares/queries.py index 023ff2fb8..591a59d27 100644 --- a/tests_new/integration_tests/modules/shares/queries.py +++ b/tests_new/integration_tests/modules/shares/queries.py @@ -272,3 +272,34 @@ def get_s3_consumption_data(client, shareUri: str): response = client.query(query=query) return response.data.getS3ConsumptionData + + +def reapply_share_object_items(client, dataset_uri: str): + query = { + 'operationName': 'reApplyShareObjectItemsOnDataset', + 'variables': {'input': dataset_uri}, + 'query': f""" + mutation reApplyShareObjectItemsOnDataset($datasetUri: String!) {{ + reApplyShareObjectItemsOnDataset(datasetUri: $datasetUri) + }} + """, + } + response = client.query(query=query) + return response.data.reApplyShareObjectItemsOnDataset + + +def reapply_items_share_object(client, share_uri: str, item_uris: List[str]): + query = { + 'operationName': 'reApplyItemsShareObject', + 'variables': {'input': {'shareUri': share_uri, 'itemUris': item_uris}}, + 'query': f""" + mutation reApplyItemsShareObject($input: ShareItemSelectorInput) {{ + reApplyItemsShareObject(input: $input) {{ + shareUri + status + }} + }} + """, + } + response = client.query(query=query) + return response.data.reApplyItemsShareObject diff --git a/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py b/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py index f195a545f..535480897 100644 --- a/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py +++ b/tests_new/integration_tests/modules/shares/s3_datasets_shares/conftest.py @@ -253,15 +253,15 @@ def persistent_consumption_role_1(client5, group5, persistent_cross_acc_env_1, p def persistent_group_share_1( client5, client1, - updated_persistent_env1, - updated_persistent_cross_acc_env_1, + persistent_env1, + persistent_cross_acc_env_1, updated_persistent_s3_dataset1, group5, ): share1 = create_share_object( client=client5, dataset_or_item_params={'datasetUri': updated_persistent_s3_dataset1.datasetUri}, - environmentUri=updated_persistent_cross_acc_env_1.environmentUri, + environmentUri=persistent_cross_acc_env_1.environmentUri, groupUri=group5, principalId=group5, principalType='Group', @@ -285,8 +285,8 @@ def persistent_group_share_1( def persistent_role_share_1( client5, client1, - updated_persistent_env1, - updated_persistent_cross_acc_env_1, + persistent_env1, + persistent_cross_acc_env_1, updated_persistent_s3_dataset1, group5, persistent_consumption_role_1, @@ -294,7 +294,7 @@ def persistent_role_share_1( share1 = create_share_object( client=client5, dataset_or_item_params={'datasetUri': updated_persistent_s3_dataset1.datasetUri}, - environmentUri=updated_persistent_cross_acc_env_1.environmentUri, + environmentUri=persistent_cross_acc_env_1.environmentUri, groupUri=group5, principalId=persistent_consumption_role_1.consumptionRoleUri, principalType='ConsumptionRole', diff --git a/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py b/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py index 294aa9e82..b28a5bdeb 100644 --- a/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py +++ b/tests_new/integration_tests/modules/shares/s3_datasets_shares/shared_test_functions.py @@ -84,7 +84,7 @@ def check_share_succeeded(client, shareUri, check_contains_all_item_types=False) assert_that(items).extracting('itemType').contains(*ALL_S3_SHARABLE_TYPES_NAMES) -def check_verify_share_items(client, shareUri): +def check_verify_share_items(client, shareUri, expected_health_status=['Healthy'], expected_health_msg=[]): share = get_share_object(client, shareUri, {'isShared': True}) items = share['items'].nodes times = [item.lastVerificationTime for item in items] @@ -93,8 +93,11 @@ def check_verify_share_items(client, shareUri): updated_share = get_share_object(client, shareUri, {'isShared': True}) items = updated_share['items'].nodes assert_that(items).extracting('status').contains_only('Share_Succeeded') - assert_that(items).extracting('healthStatus').contains_only('Healthy') + assert_that(items).extracting('healthStatus').contains_only(*expected_health_status) assert_that(items).extracting('lastVerificationTime').does_not_contain(*times) + if expected_health_msg: + health_msgs = ' '.join([i.healthMessage for i in items]) + assert_that(health_msgs).contains(*expected_health_msg) def check_table_access( diff --git a/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py b/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py index bae4e0ad9..e54e04db4 100644 --- a/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py +++ b/tests_new/integration_tests/modules/shares/s3_datasets_shares/test_new_crossacc_s3_share.py @@ -1,6 +1,8 @@ import pytest from assertpy import assert_that +from integration_tests.errors import GqlError +from integration_tests.modules.s3_datasets.aws_clients import LakeFormationClient from integration_tests.modules.shares.s3_datasets_shares.conftest import clean_up_share from tests_new.integration_tests.modules.shares.queries import ( create_share_object, @@ -11,12 +13,9 @@ delete_share_object, update_share_request_reason, update_share_reject_reason, + reapply_items_share_object, ) -from tests_new.integration_tests.modules.shares.utils import ( - check_share_ready, -) - -from integration_tests.modules.shares.s3_datasets_shares.shared_test_functions import ( +from tests_new.integration_tests.modules.shares.s3_datasets_shares.shared_test_functions import ( check_share_items_access, check_verify_share_items, revoke_and_check_all_shared_items, @@ -26,6 +25,10 @@ check_approve_share_object, check_share_succeeded, ) +from tests_new.integration_tests.modules.shares.utils import ( + check_share_ready, + check_share_items_reapplied, +) def test_create_and_delete_share_object(client5, session_cross_acc_env_1, session_s3_dataset1, principal1, group5): @@ -176,6 +179,58 @@ def test_check_item_access( ) +@pytest.mark.dependency(name='unhealthy_items', depends=['share_verified']) +def test_unhealthy_items( + client5, session_cross_acc_env_1_aws_client, session_cross_acc_env_1_integration_role_arn, share_params_main +): + share, _ = share_params_main + iam = session_cross_acc_env_1_aws_client.resource('iam') + principal_role = iam.Role(share.principal.principalRoleName) + # break s3 by removing policies + for policy in principal_role.attached_policies.all(): + if '/dataall-env-' in policy.arn and 'share-policy' in policy.arn: + principal_role.detach_policy(PolicyArn=policy.arn) + # break lf by removing DESCRIBE perms from principal + lf_client = LakeFormationClient(session_cross_acc_env_1_aws_client, session_cross_acc_env_1_aws_client.region_name) + lf_client.add_role_to_datalake_admin(session_cross_acc_env_1_integration_role_arn) + db_name = f'dataall_{share.dataset.datasetName}_{share.dataset.datasetUri}_shared'.replace('-', '_') + lf_client.revoke_db_perms(principal_role.arn, db_name, ['DESCRIBE']) + # verify all items are `Unhealthy` + check_verify_share_items( + client5, + share.shareUri, + expected_health_status=['Unhealthy'], + expected_health_msg=[ + 'IAM Policy attached Target Resource does not exist', + 'missing LF permissions: DESCRIBE', + ], + ) + + +@pytest.mark.dependency(depends=['share_approved']) +def test_reapply_unauthoried(client5, share_params_main): + share, _ = share_params_main + share_uri = share.shareUri + share_object = get_share_object(client5, share_uri) + item_uris = [item.shareItemUri for item in share_object['items'].nodes] + assert_that(reapply_items_share_object).raises(GqlError).when_called_with(client5, share_uri, item_uris).contains( + 'UnauthorizedOperation' + ) + + +@pytest.mark.dependency(depends=['share_approved']) +def test_reapply(client1, share_params_main): + share, _ = share_params_main + share_uri = share.shareUri + share_object = get_share_object(client1, share_uri) + item_uris = [item.shareItemUri for item in share_object['items'].nodes] + reapply_items_share_object(client1, share_uri, item_uris) + share_object = get_share_object(client1, share_uri) + assert_that(share_object['items'].nodes).extracting('healthStatus').contains_only('PendingReApply') + check_share_items_reapplied(client1, share_uri) + check_verify_share_items(client1, share_uri) + + @pytest.mark.dependency(name='share_revoked', depends=['share_succeeded']) def test_revoke_share(client1, share_params_main): share, dataset = share_params_main