Integration tests executed on a real deployment as part of the CICD -…

… Redshift Shares (#1643)

⚠️ MERGE AFTER #1636 
### Feature or Bugfix
- Feature: Testing

### Detail
Add integration tests for Redshift shares. Implements #1620 
- Implemented inside the shares modules in a subdirectory so that each
share type can have its own conftest but still re-use common methods
from shares queries
- This PR is focused on testing the Redshift shares functionality, it
does not include all tests that test the workflow of the share (e.g.
submit, reject...)
- It does not validate if after a share the user has access to data. We
could implement it using the Redshift Data API, but I left it as
optional for a separate PR

### Tested
Locally:

![image](https://github.com/user-attachments/assets/3a2acc79-d025-483f-949b-23e31b23d26e)


### Relates
- #1620
- #1619
- #1220

### Security
Please answer the questions below briefly where applicable, or write
`N/A`. Based on
[OWASP 10](https://owasp.org/Top10/en/).

- Does this PR introduce or modify any input fields or queries - this
includes
fetching data from storage outside the application (e.g. a database, an
S3 bucket)?
  - Is the input sanitized?
- What precautions are you taking before deserializing the data you
consume?
  - Is injection prevented by parametrizing queries?
  - Have you ensured no `eval` or similar functions are used?
- Does this PR introduce any functionality or component that requires
authorization?
- How have you ensured it respects the existing AuthN/AuthZ mechanisms?
  - Are you logging failed auth attempts?
- Are you using or adding any cryptographic features?
  - Do you use a standard proven implementations?
  - Are the used keys controlled by the customer? Where are they stored?
- Are you introducing any new policies/roles/users?
  - Have you used the least-privilege principle? How?


By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

backend/dataall/core/environment/cdk/environment_stack.py

@@ -688,3 +688,11 @@ def create_integration_tests_role(self):
                 resources=[f'arn:aws:quicksight:*:{self.account}:*'],
             ),
         )
+
+        self.test_role.add_to_policy(
+            iam.PolicyStatement(
+                actions=['redshift:DeauthorizeDataShare'],
+                effect=iam.Effect.ALLOW,
+                resources=[f'arn:aws:redshift:{self.region}:{self.account}:datashare:*/dataall*'],
+            ),
+        )

tests_new/integration_tests/conftest.py

@@ -54,8 +54,8 @@ class RedshiftConnection:
 class TestData:
     users: dict[str, User]
     envs: dict[str, Env]
-    dashboards: dict[str, Dashboard]
-    redshift_connections: dict[str, RedshiftConnection]
+    dashboards: dict[str, Dashboard] = None
+    redshift_connections: dict[str, RedshiftConnection] = None
 
 
 @pytest.fixture(scope='session', autouse=True)

tests_new/integration_tests/modules/redshift_datasets/connection_queries.py

@@ -162,6 +162,8 @@ def create_redshift_connection(
                 connectionUri
                 connectionType
                 redshiftType
+                name
+                nameSpaceId
               }
             }
         """,

tests_new/integration_tests/modules/redshift_datasets/global_conftest.py

@@ -14,6 +14,7 @@
     import_redshift_dataset,
     delete_redshift_dataset,
     list_redshift_dataset_tables,
+    add_redshift_dataset_tables,
 )
 
 log = logging.getLogger(__name__)
@@ -24,6 +25,13 @@
 REDSHIFT_TABLE2 = 'nation'
 
 
+@pytest.fixture(scope='session')
+def redshift_connections(testdata):
+    if testdata.redshift_connections:
+        return testdata.redshift_connections
+    pytest.skip('redshift config is missing')
+
+
 def create_connection(client, env, group, name, conn_type, red_type, connection_data=RedshiftConnection):
     connection = create_redshift_connection(
         client=client,
@@ -64,7 +72,7 @@ def create_connection(client, env, group, name, conn_type, red_type, connection_
 
 
 @pytest.fixture(scope='session')
-def session_connection_serverless_admin(client1, group1, session_env1, testdata):
+def session_connection_serverless_admin(client1, group1, session_env1, redshift_connections):
     connection = None
     try:
         connection = create_connection(
@@ -74,7 +82,7 @@ def session_connection_serverless_admin(client1, group1, session_env1, testdata)
             env=session_env1,
             group=group1,
             red_type='serverless',
-            connection_data=testdata.redshift_connections['connection_serverless_admin_session_env1'],
+            connection_data=redshift_connections['connection_serverless_admin_session_env1'],
         )
 
         yield connection
@@ -102,7 +110,7 @@ def session_connection_serverless_admin_group_with_permissions(client1, group5,
 
 
 @pytest.fixture(scope='session')
-def session_connection_serverless_data_user(client1, group1, session_env1, testdata):
+def session_connection_serverless_data_user(client1, group1, session_env1, redshift_connections):
     connection = None
     try:
         connection = create_connection(
@@ -112,7 +120,7 @@ def session_connection_serverless_data_user(client1, group1, session_env1, testd
             env=session_env1,
             group=group1,
             red_type='serverless',
-            connection_data=testdata.redshift_connections['connection_serverless_data_user_session_env1'],
+            connection_data=redshift_connections['connection_serverless_data_user_session_env1'],
         )
         yield connection
     finally:
@@ -121,7 +129,7 @@ def session_connection_serverless_data_user(client1, group1, session_env1, testd
 
 
 @pytest.fixture(scope='session')
-def session_connection_cluster_admin(client5, group5, session_cross_acc_env_1, testdata):
+def session_connection_cluster_admin(client5, group5, session_cross_acc_env_1, redshift_connections):
     connection = None
     try:
         connection = create_connection(
@@ -131,7 +139,7 @@ def session_connection_cluster_admin(client5, group5, session_cross_acc_env_1, t
             env=session_cross_acc_env_1,
             group=group5,
             red_type='cluster',
-            connection_data=testdata.redshift_connections['connection_cluster_admin_session_cross_acc_env_1'],
+            connection_data=redshift_connections['connection_cluster_admin_session_cross_acc_env_1'],
         )
         yield connection
     finally:
@@ -140,7 +148,7 @@ def session_connection_cluster_admin(client5, group5, session_cross_acc_env_1, t
 
 
 @pytest.fixture(scope='session')
-def session_connection_cluster_data_user(client5, group5, session_cross_acc_env_1, testdata):
+def session_connection_cluster_data_user(client5, group5, session_cross_acc_env_1, redshift_connections):
     connection = None
     try:
         connection = create_connection(
@@ -150,7 +158,7 @@ def session_connection_cluster_data_user(client5, group5, session_cross_acc_env_
             env=session_cross_acc_env_1,
             group=group5,
             red_type='cluster',
-            connection_data=testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'],
+            connection_data=redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'],
         )
         yield connection
     finally:
@@ -218,3 +226,14 @@ def session_redshift_dataset_cluster(
     finally:
         if dataset:
             delete_redshift_dataset(client=client5, dataset_uri=dataset.datasetUri)
+
+
+@pytest.fixture(scope='session')
+def session_redshift_dataset_cluster_table(client5, session_redshift_dataset_cluster):
+    add_redshift_dataset_tables(
+        client=client5, dataset_uri=session_redshift_dataset_cluster.datasetUri, tables=[REDSHIFT_TABLE1]
+    )
+    tables = list_redshift_dataset_tables(
+        client=client5, dataset_uri=session_redshift_dataset_cluster.datasetUri, term=REDSHIFT_TABLE1
+    )
+    yield tables.nodes[0]

tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py

@@ -32,8 +32,8 @@ def test_create_connection(connection_fixture_name, connection_type, redshift_ty
     assert_that(connection.redshiftType).is_equal_to(redshift_type)
 
 
-def test_create_serverless_connection_namespace_does_not_exist(client1, group1, session_env1, testdata):
-    connection_data = testdata.redshift_connections['connection_serverless_data_user_session_env1']
+def test_create_serverless_connection_namespace_does_not_exist(client1, group1, session_env1, redshift_connections):
+    connection_data = redshift_connections['connection_serverless_data_user_session_env1']
     error_namespace_id = 'doesnotexist'
     assert_that(create_redshift_connection).raises(GqlError).when_called_with(
         client=client1,
@@ -50,8 +50,8 @@ def test_create_serverless_connection_namespace_does_not_exist(client1, group1,
     ).contains('Redshift namespaceId', error_namespace_id, 'not exist')
 
 
-def test_create_serverless_connection_workgroup_not_found(client1, group1, session_env1, testdata):
-    connection_data = testdata.redshift_connections['connection_serverless_data_user_session_env1']
+def test_create_serverless_connection_workgroup_not_found(client1, group1, session_env1, redshift_connections):
+    connection_data = redshift_connections['connection_serverless_data_user_session_env1']
     error_workgroup = 'doesnotexist'
     assert_that(create_redshift_connection).raises(GqlError).when_called_with(
         client=client1,
@@ -68,8 +68,8 @@ def test_create_serverless_connection_workgroup_not_found(client1, group1, sessi
     ).contains('Redshift workgroup', error_workgroup, 'not exist')
 
 
-def test_create_cluster_connection_cluster_not_found(client5, group5, session_cross_acc_env_1, testdata):
-    connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
+def test_create_cluster_connection_cluster_not_found(client5, group5, session_cross_acc_env_1, redshift_connections):
+    connection_data = redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
     error_cluster_id = 'doesnotexist'
     assert_that(create_redshift_connection).raises(GqlError).when_called_with(
         client=client5,
@@ -90,8 +90,8 @@ def test_create_cluster_connection_cluster_not_encrypted():
     pass
 
 
-def test_create_connection_database_not_found(client5, group5, session_cross_acc_env_1, testdata):
-    connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
+def test_create_connection_database_not_found(client5, group5, session_cross_acc_env_1, redshift_connections):
+    connection_data = redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
     error_database = 'doesnotexist'
     assert_that(create_redshift_connection).raises(GqlError).when_called_with(
         client=client5,
@@ -107,8 +107,8 @@ def test_create_connection_database_not_found(client5, group5, session_cross_acc
     ).contains('Redshift database', error_database, 'not exist')
 
 
-def test_create_connection_unauthorized(client1, group1, session_cross_acc_env_1, testdata):
-    connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
+def test_create_connection_unauthorized(client1, group1, session_cross_acc_env_1, redshift_connections):
+    connection_data = redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
     assert_that(create_redshift_connection).raises(GqlError).when_called_with(
         client=client1,
         connection_name='errorConnection',
@@ -123,8 +123,8 @@ def test_create_connection_unauthorized(client1, group1, session_cross_acc_env_1
     ).contains('UnauthorizedOperation', 'CREATE_REDSHIFT_CONNECTION', session_cross_acc_env_1.environmentUri)
 
 
-def test_delete_connection(client5, group5, session_cross_acc_env_1, testdata):
-    connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
+def test_delete_connection(client5, group5, session_cross_acc_env_1, redshift_connections):
+    connection_data = redshift_connections['connection_cluster_data_user_session_cross_acc_env_1']
     connection = create_redshift_connection(
         client=client5,
         connection_name='errorConnection',

tests_new/integration_tests/modules/share_base/queries.py → tests_new/integration_tests/modules/shares/queries.py

@@ -1,5 +1,4 @@
-from tests_new.integration_tests.modules.share_base.input_types import NewShareObjectInput
-from tests_new.integration_tests.modules.share_base.types import ShareObject
+from tests_new.integration_tests.modules.shares.types import ShareObject
 from typing import List
 
 
@@ -9,18 +8,29 @@ def create_share_object(
     environmentUri,
     groupUri,
     principalId,
+    principalRoleName,
     principalType,
     requestPurpose,
     attachMissingPolicies,
     permissions,
 ):
-    variables = dataset_or_item_params
-    variables['input'] = NewShareObjectInput(
-        environmentUri, groupUri, principalId, principalType, requestPurpose, attachMissingPolicies, permissions
-    )
     query = {
         'operationName': 'createShareObject',
-        'variables': variables,
+        'variables': {
+            'datasetUri': dataset_or_item_params.get('datasetUri'),
+            'itemType': dataset_or_item_params.get('itemType'),
+            'itemUri': dataset_or_item_params.get('itemUri'),
+            'input': {
+                'environmentUri': environmentUri,
+                'groupUri': groupUri,
+                'principalId': principalId,
+                'principalRoleName': principalRoleName,
+                'principalType': principalType,
+                'requestPurpose': requestPurpose,
+                'attachMissingPolicies': attachMissingPolicies,
+                'permissions': permissions,
+            },
+        },
         'query': f"""
                     mutation createShareObject(        $datasetUri: String!
         $itemType: String
@@ -209,6 +219,24 @@ def verify_share_items(client, shareUri: str, shareItemsUris: List[str]):
     return response.data.verifyItemsShareObject
 
 
+def reapply_share_items(client, shareUri: str, shareItemsUris: List[str]):
+    query = {
+        'operationName': 'reApplyItemsShareObject',
+        'variables': {'input': {'shareUri': shareUri, 'itemUris': shareItemsUris}},
+        'query': f"""
+                mutation reApplyItemsShareObject($input: ShareItemSelectorInput) {{
+                  reApplyItemsShareObject(input: $input) {{
+                    shareUri
+                    status
+                  }}
+                }}
+                """,
+    }
+
+    response = client.query(query=query)
+    return response.data.reApplyItemsShareObject
+
+
 def revoke_share_items(client, shareUri: str, shareItemUris: List[str]):
     query = {
         'operationName': 'revokeItemsShareObject',

tests_new/integration_tests/modules/shares/redshift_datasets_shares/aws_clients.py

@@ -0,0 +1,23 @@
+import logging
+from typing import Any, Dict
+from botocore.exceptions import ClientError
+
+log = logging.getLogger(__name__)
+
+
+class RedshiftClient:
+    def __init__(self, session, region):
+        self._client = session.client('redshift', region_name=region)
+        self._region = region
+
+    def deauthorize_datashare(self, datashare_arn: str, target_account: str) -> Dict[str, Any]:
+        log.info('Deauthorizing Redshift datashare...')
+        try:
+            response = self._client.deauthorize_data_share(
+                DataShareArn=datashare_arn, ConsumerIdentifier=target_account
+            )
+            log.info(f'Datashare deauthorized successfully: {datashare_arn}')
+            return response
+        except ClientError as e:
+            log.exception('Error deauthorizing datashare')
+            raise e