CICD Integration tests: new shares for pre-existing datasets (#1611)

<!-- please choose -->
- Feature

For group share and consumption role shares perform

- [x] Create Share request
- [x] Submit request without Items (not allowed)
- [x] Add items to request
- [x] Submit request
- [x] Change request purpose
- [x] Reject request
- [x] Change reject purpose
- [x] Approve request
- [x] Request succeeded
- [x] Item health verification
- [x] Folder sharing validation (cal list objects via s3 accesspoint)
- [x] Bucket sharing validation (cal list objects in s3 bucket )
- [x] Table sharing validation (cal perform athena query: select * from
db.table )
- [x] Revoke items
- [x] Folder sharing validation (no more access)
- [x] Bucket sharing validation (no more access)
- [x] Table sharing validation (no more access)
- [x] Delete share

Tests are the same as for new shares for new datasets
Fixtures are parametrised

- #1376

Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).

- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
  - Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
  - Is injection prevented by parametrizing queries?
  - Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
  - Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
  - Do you use a standard proven implementations?
  - Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
  - Have you used the least-privilege principle? How?

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Sofia Sazonova <[email protected]>

backend/dataall/core/environment/cdk/environment_stack.py

@@ -596,13 +596,20 @@ def create_integration_tests_role(self):
                     's3:CreateBucket',
                     's3:DeleteBucket',
                     's3:PutEncryptionConfiguration',
-                    's3:List*',
                     's3:GetObject*',
                     's3:DeleteObject',
+                    's3:DeleteObjectVersion',
                 ],
                 effect=iam.Effect.ALLOW,
                 resources=[
                     'arn:aws:s3:::dataalltesting*',
+                    'arn:aws:s3:::dataalltesting*/*',
+                    'arn:aws:s3:::dataall-session*',
+                    'arn:aws:s3:::dataall-session*/*',
+                    'arn:aws:s3:::dataall-test-session*',
+                    'arn:aws:s3:::dataall-test-session*/*',
+                    'arn:aws:s3:::dataall-temp*',
+                    'arn:aws:s3:::dataall-temp*/*',
                     'arn:aws:s3:::dataall-env-access-logs*',
                 ],
             )
@@ -623,8 +630,10 @@ def create_integration_tests_role(self):
             iam.PolicyStatement(
                 actions=[
                     'lakeformation:GrantPermissions',
+                    'lakeformation:RevokePermissions',
                     'lakeformation:PutDataLakeSettings',
                     'lakeformation:GetDataLakeSettings',
+                    'glue:GetDatabase',
                     'kms:CreateKey',
                     'kms:CreateAlias',
                     'kms:DeleteAlias',
@@ -633,7 +642,11 @@ def create_integration_tests_role(self):
                     'kms:PutKeyPolicy',
                     'kms:ScheduleKeyDeletion',
                     'kms:TagResource',
+                    'kms:DescribeKey',
                     's3:GetBucketVersioning',
+                    's3:List*',
+                    's3:ListAccessPoints',
+                    's3:DeleteAccessPoint',
                 ],
                 effect=iam.Effect.ALLOW,
                 resources=['*'],
@@ -670,3 +683,40 @@ def create_integration_tests_role(self):
                 resources=[f'arn:aws:cloudformation:*:{self.account}:stack/*/*'],
             ),
         )
+
+        self.test_role.add_to_policy(
+            iam.PolicyStatement(
+                actions=[
+                    'iam:GetRole',
+                    'iam:CreateRole',
+                    'iam:DeleteRole',
+                    'iam:PutRolePolicy',
+                    'iam:DeleteRolePolicy',
+                    'iam:DetachRolePolicy',
+                    'iam:ListAttachedRolePolicies',
+                ],
+                effect=iam.Effect.ALLOW,
+                resources=[
+                    f'arn:aws:iam::{self.account}:role/dataall-test*',
+                    f'arn:aws:iam::{self.account}:role/dataall-session*',
+                ],
+            ),
+        )
+
+        self.test_role.add_to_policy(
+            iam.PolicyStatement(
+                actions=[
+                    'quicksight:DescribeAccountSubscription',
+                ],
+                effect=iam.Effect.ALLOW,
+                resources=[f'arn:aws:quicksight:*:{self.account}:*'],
+            ),
+        )
+
+        self.test_role.add_to_policy(
+            iam.PolicyStatement(
+                actions=['redshift:DeauthorizeDataShare'],
+                effect=iam.Effect.ALLOW,
+                resources=[f'arn:aws:redshift:{self.region}:{self.account}:datashare:*/dataall*'],
+            ),
+        )

tests_new/integration_tests/aws_clients/athena.py

@@ -0,0 +1,47 @@
+import time
+from tests_new.integration_tests.utils import poller
+
+
+class AthenaClient:
+    def __init__(self, session, region):
+        self._client = session.client('athena', region_name=region)
+        self._region = region
+
+    def _run_query(self, query, workgroup='primary', output_location=None):
+        if output_location:
+            result = self._client.start_query_execution(
+                QueryString=query, ResultConfiguration={'OutputLocation': output_location}
+            )
+        else:
+            result = self._client.start_query_execution(QueryString=query, WorkGroup=workgroup)
+        return result['QueryExecutionId']
+
+    @poller(check_success=lambda state: state not in ['QUEUED', 'RUNNING'], timeout=600, sleep_time=5)
+    def _wait_for_query(self, query_id):
+        result = self._client.get_query_execution(QueryExecutionId=query_id)
+        return result['QueryExecution']['Status']['State']
+
+    def execute_query(self, query, workgroup='primary', output_location=None):
+        q_id = self._run_query(query, workgroup, output_location)
+        return self._wait_for_query(q_id)
+
+    def list_work_groups(self):
+        result = self._client.list_work_groups()
+        return [x['Name'] for x in result['WorkGroups']]
+
+    def get_work_group(self, env_name, group):
+        workgroups = self.list_work_groups()
+        group_snakify = group.lower().replace('-', '_')
+        env_name_snakify = env_name.lower().replace('-', '_')
+        default_workgroup = 'primary'
+        env_workgroup, group_workgroup = None, None
+        for workgroup in workgroups:
+            if env_name_snakify in workgroup:
+                env_workgroup = workgroup
+            if group_snakify in workgroup:
+                group_workgroup = workgroup
+        if group_workgroup:
+            return group_workgroup
+        elif env_workgroup:
+            return env_workgroup
+        return default_workgroup

tests_new/integration_tests/aws_clients/iam.py

@@ -0,0 +1,81 @@
+import json
+import logging
+import os
+
+import boto3
+
+log = logging.getLogger(__name__)
+
+
+class IAMClient:
+    CONSUMPTION_POLICY_NAME = 'ConsumptionPolicy'
+
+    def __init__(self, session=boto3.Session(), region=os.environ.get('AWS_REGION', 'us-east-1')):
+        self._client = session.client('iam', region_name=region)
+        self._resource = session.resource('iam', region_name=region)
+        self._region = region
+
+    def get_role(self, role_name):
+        try:
+            role = self._client.get_role(RoleName=role_name)
+            return role
+        except self._client.exceptions.NoSuchEntityException as e:
+            log.info(f'Error occurred: {e}')
+            return None
+
+    def delete_role(self, role_name):
+        self._client.delete_role(RoleName=role_name)
+
+    def create_role(self, account_id, role_name, test_role_name):
+        policy_doc = {
+            'Version': '2012-10-17',
+            'Statement': [
+                {
+                    'Effect': 'Allow',
+                    'Principal': {'AWS': [f'arn:aws:iam::{account_id}:role/{test_role_name}']},
+                    'Action': 'sts:AssumeRole',
+                }
+            ],
+        }
+        role = self._client.create_role(
+            RoleName=role_name,
+            AssumeRolePolicyDocument=json.dumps(policy_doc),
+            Description='Role for Lambda function',
+        )
+        return role
+
+    def get_consumption_role(self, account_id, role_name, test_role_name):
+        role = self.get_role(role_name)
+        if role is None:
+            role = self.create_role(account_id, role_name, test_role_name)
+            self.put_consumption_role_policy(role_name)
+        return role
+
+    def delete_policy(self, role_name, policy_name):
+        self._client.delete_role_policy(RoleName=role_name, PolicyName=policy_name)
+
+    def delete_consumption_role(self, role_name):
+        self.delete_policy(role_name, self.CONSUMPTION_POLICY_NAME)
+        self.delete_role(role_name)
+
+    def put_consumption_role_policy(self, role_name):
+        self._client.put_role_policy(
+            RoleName=role_name,
+            PolicyName=self.CONSUMPTION_POLICY_NAME,
+            PolicyDocument="""{
+                                        "Version": "2012-10-17",
+                                        "Statement": [
+                                            {
+                                                "Sid": "TestPolicyDoc",
+                                                "Effect": "Allow",
+                                                "Action": [
+                                                    "s3:*",
+                                                    "athena:*",
+                                                    "glue:*",
+                                                    "lakeformation:GetDataAccess"
+                                                ],
+                                                "Resource": "*"
+                                            }
+                                        ]
+                                    }""",
+        )

tests_new/integration_tests/conftest.py

@@ -16,6 +16,7 @@
     'integration_tests.core.organizations.global_conftest',
     'integration_tests.core.environment.global_conftest',
     'integration_tests.modules.s3_datasets.global_conftest',
+    'integration_tests.modules.redshift_datasets.global_conftest',
 ]
 
 
@@ -33,11 +34,28 @@ class Env:
     region: str
 
 
+@dataclass_json
+@dataclass
+class Dashboard:
+    dashboardId: str
+
+
+@dataclass_json
+@dataclass
+class RedshiftConnection:
+    secret_arn: str
+    namespace_id: str = None
+    workgroup: str = None
+    cluster_id: str = None
+
+
 @dataclass_json
 @dataclass
 class TestData:
     users: dict[str, User]
     envs: dict[str, Env]
+    dashboards: dict[str, Dashboard] = None
+    redshift_connections: dict[str, RedshiftConnection] = None
 
 
 @pytest.fixture(scope='session', autouse=True)
@@ -84,6 +102,18 @@ def user4(userdata):
     yield userdata['testUser4']
 
 
+@pytest.fixture(scope='session', autouse=True)
+def user5(userdata):
+    # Existing user with name and password
+    yield userdata['testUser5']
+
+
+@pytest.fixture(scope='session', autouse=True)
+def user6(userdata):
+    # Existing user with name and password
+    yield userdata['testUser6']
+
+
 @pytest.fixture(scope='session', autouse=True)
 def group1():
     # Existing Cognito group with name testGroup1
@@ -112,6 +142,20 @@ def group4():
     yield 'testGroup4'
 
 
+@pytest.fixture(scope='session', autouse=True)
+def group5():
+    # Existing Cognito group with name testGroup5
+    # Add user5
+    yield 'testGroup5'
+
+
+@pytest.fixture(scope='session', autouse=True)
+def group6():
+    # Existing Cognito group with name testGroup5
+    # Add user5
+    yield 'testGroup6'
+
+
 @pytest.fixture(scope='session')
 def client1(user1) -> Client:
     yield Client(user1.username, user1.password)
@@ -132,6 +176,16 @@ def client4(user4) -> Client:
     yield Client(user4.username, user4.password)
 
 
+@pytest.fixture(scope='session')
+def client5(user5) -> Client:
+    yield Client(user5.username, user5.password)
+
+
+@pytest.fixture(scope='session')
+def client6(user6) -> Client:
+    yield Client(user6.username, user6.password)
+
+
 @pytest.fixture(scope='session')
 def clientTenant(userTenant) -> Client:
     yield Client(userTenant.username, userTenant.password)