Integration tests executed on a real deployment as part of the CICD - Redshift Connections

backend/dataall/modules/redshift_datasets/aws/redshift.py

@@ -14,7 +14,14 @@ def __init__(self, account_id: str, region: str) -> None:
 
     def describe_cluster(self, clusterId: str):
         log.info(f'Describing cluster {clusterId=}')
-        return self.client.describe_clusters(ClusterIdentifier=clusterId)['Clusters'][0]
+        try:
+            return self.client.describe_clusters(ClusterIdentifier=clusterId)['Clusters'][0]
+        except ClientError as e:
+            if e.response['Error']['Code'] == 'ClusterNotFound':
+                log.error(f'Redshift cluster {clusterId} does not exist')
+                return None
+            else:
+                raise e
 
     def get_cluster_namespaceId(self, clusterId: str):
         log.info(f'Describing cluster {clusterId=}')

backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py

@@ -76,12 +76,10 @@ def get_statements(self):
                 cluster_arns = [
                     f'arn:aws:redshift:{self.region}:{self.account}:cluster:{conn.clusterId}'
                     for conn in connections
-                    if conn.clusterId != ''
+                    if conn.clusterId
                 ]
                 workgroup_arns = [
-                    rs_client.get_workgroup_arn(workgroup_name=conn.workgroup)
-                    for conn in connections
-                    if conn.workgroup != ''
+                    rs_client.get_workgroup_arn(workgroup_name=conn.workgroup) for conn in connections if conn.workgroup
                 ]
                 additional_statements.extend(
                     split_policy_with_resources_in_statements(

backend/dataall/modules/redshift_datasets/db/redshift_connection_repositories.py

@@ -128,8 +128,6 @@ def _query_redshift_connection_group_permissions(session, connection_uri, permis
             .group_by(ResourcePolicy.principalId)
         )
 
-        if filter and filter.get('groupUri'):
-            query = query.filter(ResourcePolicy.principalId == filter.get('groupUri'))
         if filter and filter.get('term'):
             query = query.filter(
                 ResourcePolicy.principalId.ilike(filter.get('term') + '%%'),

backend/dataall/modules/redshift_datasets/services/redshift_connection_service.py

@@ -113,6 +113,7 @@ def delete_redshift_connection(uri) -> bool:
             )
             session.delete(connection)
             session.commit()
+        StackService.deploy_stack(targetUri=connection.environmentUri)
         return True
 
     @staticmethod

tests_new/integration_tests/README.md

@@ -55,7 +55,7 @@ Currently **we support only Cognito based deployments** but support for any IdP
                 ]
               }
             },
-            "envs": {
+              "envs": {
               "session_env1": {
                 "accountId": "...",
                 "region": "eu-central-1"
@@ -64,24 +64,87 @@ Currently **we support only Cognito based deployments** but support for any IdP
                 "accountId": "...",
                 "region": "eu-west-1"
               },
-               "persistent_env1": {
-                       "accountId": "...",
-                       "region": "us-east-1"
-               },
+              "persistent_env1": {
+                "accountId": "...",
+                "region": "us-east-1"
+              },
+               "persistent_cross_acc_env_1": {
+                  "accountId": "...",
+                  "region": "us-east-1"
+                },
               "session_cross_acc_env_1": {
-                       "accountId": "another acc",
-                       "region": "same as session_env1"  
-               },
+                "accountId": "...",
+                "region": "eu-central-1"
+              }
             },
             "dashboards": {
                 "session_env1": {
                   "dashboardId": "..."
                 },
+              },
+             "redshift_connections": {
+              "connection_serverless_admin_session_env1": {
+                "namespace_id": "...",
+                "workgroup": "...",
+                "secret_arn": "..."
+              },
+              "connection_serverless_data_user_session_env1": {
+                "namespace_id": "...",
+                "workgroup": "...",
+                "secret_arn": "..."
+              },
+              "connection_cluster_admin_session_cross_acc_env_1": {
+                "cluster_id": "...",
+                "secret_arn": "..."
+              },
+              "connection_cluster_data_admin_session_cross_acc_env_1": {
+                "cluster_id": "...",
+                "secret_arn": "..."
               }
+            }
           }
           ```
 
 - The pipeline will create the users/groups
+- For Redshift testing we require some pre-existing infrastructure:
+  - One Redshift serverless namespace+workgroup and one Redshift provisioned cluster - both MUST be encrypted
+  - Both must host the default `dev` database with the `public` schema.
+  - For each we need to ensure that the admin credentials are stored in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True}
+  - For each we need to create a Redshift user (see SQL commands below) and store the credentials in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True}
+  - For each we need to create a set of tables using the commands below
+  - For each we need to create a Redshift role ()
+
+Create User and grant basic permissions using admin connection
+```sql
+CREATE USER testuser PASSWORD 'Pass1Word!';
+GRANT USAGE ON SCHEMA public TO testuser;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO testuser;
+```
+
+Create and attach role using admin connection
+```sql
+CREATE ROLE testrole;
+GRANT ROLE testrole TO testuser;
+```
+
+Create tables using testuser connection
+```sql
+DROP TABLE IF EXISTS nation;
+DROP TABLE IF EXISTS region;
+
+CREATE TABLE region (
+  R_REGIONKEY bigint NOT NULL,
+  R_NAME varchar(25),
+  R_COMMENT varchar(152))
+diststyle all;
+
+CREATE TABLE nation (
+  N_NATIONKEY bigint NOT NULL,
+  N_NAME varchar(25),
+  N_REGIONKEY bigint,
+  N_COMMENT varchar(152))
+diststyle all;
+```
 
 ## Run tests
 

tests_new/integration_tests/conftest.py

@@ -16,6 +16,7 @@
     'integration_tests.core.organizations.global_conftest',
     'integration_tests.core.environment.global_conftest',
     'integration_tests.modules.s3_datasets.global_conftest',
+    'integration_tests.modules.redshift_datasets.global_conftest',
 ]
 
 
@@ -39,12 +40,22 @@ class Dashboard:
     dashboardId: str
 
 
+@dataclass_json
+@dataclass
+class RedshiftConnection:
+    secret_arn: str
+    namespace_id: str = None
+    workgroup: str = None
+    cluster_id: str = None
+
+
 @dataclass_json
 @dataclass
 class TestData:
     users: dict[str, User]
     envs: dict[str, Env]
     dashboards: dict[str, Dashboard]
+    redshift_connections: dict[str, RedshiftConnection]
 
 
 @pytest.fixture(scope='session', autouse=True)