From d367e3ee084b2a48a210282e896992d8a0008d0b Mon Sep 17 00:00:00 2001 From: dlpzx Date: Tue, 8 Oct 2024 09:47:06 +0200 Subject: [PATCH 1/7] RS integration test - added queries and skeleton --- tests_new/integration_tests/README.md | 79 +++- tests_new/integration_tests/conftest.py | 12 + .../modules/redshift_datasets/conftest.py | 10 + .../redshift_datasets/connection_queries.py | 226 ++++++++++++ .../redshift_datasets/dataset_queries.py | 346 ++++++++++++++++++ .../redshift_datasets/global_conftest.py | 133 +++++++ .../test_redshift_connection.py | 62 ++++ .../test_redshift_dataset.py | 0 8 files changed, 860 insertions(+), 8 deletions(-) create mode 100644 tests_new/integration_tests/modules/redshift_datasets/conftest.py create mode 100644 tests_new/integration_tests/modules/redshift_datasets/connection_queries.py create mode 100644 tests_new/integration_tests/modules/redshift_datasets/dataset_queries.py create mode 100644 tests_new/integration_tests/modules/redshift_datasets/global_conftest.py create mode 100644 tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py create mode 100644 tests_new/integration_tests/modules/redshift_datasets/test_redshift_dataset.py diff --git a/tests_new/integration_tests/README.md b/tests_new/integration_tests/README.md index 157508dd8..d9579345b 100644 --- a/tests_new/integration_tests/README.md +++ b/tests_new/integration_tests/README.md @@ -55,7 +55,7 @@ Currently **we support only Cognito based deployments** but support for any IdP ] } }, - "envs": { + "envs": { "session_env1": { "accountId": "...", "region": "eu-central-1" @@ -64,24 +64,87 @@ Currently **we support only Cognito based deployments** but support for any IdP "accountId": "...", "region": "eu-west-1" }, - "persistent_env1": { - "accountId": "...", - "region": "us-east-1" - }, + "persistent_env1": { + "accountId": "...", + "region": "us-east-1" + }, + "persistent_cross_acc_env_1": { + "accountId": "...", + "region": "us-east-1" + }, "session_cross_acc_env_1": { - "accountId": "another acc", - "region": "same as session_env1" - }, + "accountId": "...", + "region": "eu-central-1" + } }, "dashboards": { "session_env1": { "dashboardId": "..." }, + }, + "redshift_connections": { + "connection_serverless_admin_session_env1": { + "namespace_id": "...", + "workgroup": "...", + "secret_arn": "..." + }, + "connection_serverless_data_user_session_env1": { + "namespace_id": "...", + "workgroup": "...", + "secret_arn": "..." + }, + "connection_cluster_admin_session_cross_acc_env_1": { + "cluster_id": "...", + "secret_arn": "..." + }, + "connection_cluster_data_admin_session_cross_acc_env_1": { + "cluster_id": "...", + "secret_arn": "..." } + } } ``` - The pipeline will create the users/groups +- For Redshift testing we require some pre-existing infrastructure: + - One Redshift serverless namespace+workgroup and one Redshift provisioned cluster - both MUST be encrypted + - Both must host the default `dev` database with the `public` schema. + - For each we need to ensure that the admin credentials are stored in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True} + - For each we need to create a Redshift user (see SQL commands below) and store the credentials in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True} + - For each we need to create a set of tables using the commands below + - For each we need to create a Redshift role () + +Create User and grant basic permissions using admin connection +```sql +CREATE USER testuser PASSWORD 'Pass1Word!'; +GRANT USAGE ON SCHEMA public TO testuser; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO testuser; +``` + +Create and attach role using admin connection +```sql +CREATE ROLE testrole; +GRANT ROLE testrole TO testuser; +``` + +Create tables using testuser connection +```sql +DROP TABLE IF EXISTS nation; +DROP TABLE IF EXISTS region; + +CREATE TABLE region ( + R_REGIONKEY bigint NOT NULL, + R_NAME varchar(25), + R_COMMENT varchar(152)) +diststyle all; + +CREATE TABLE nation ( + N_NATIONKEY bigint NOT NULL, + N_NAME varchar(25), + N_REGIONKEY bigint, + N_COMMENT varchar(152)) +diststyle all; +``` ## Run tests diff --git a/tests_new/integration_tests/conftest.py b/tests_new/integration_tests/conftest.py index 89464e029..ad9695b75 100644 --- a/tests_new/integration_tests/conftest.py +++ b/tests_new/integration_tests/conftest.py @@ -3,6 +3,7 @@ import os import re import sys +from typing import Optional from dataclasses import dataclass import pytest @@ -16,6 +17,7 @@ 'integration_tests.core.organizations.global_conftest', 'integration_tests.core.environment.global_conftest', 'integration_tests.modules.s3_datasets.global_conftest', + 'integration_tests.modules.redshift_datasets.global_conftest', ] @@ -39,12 +41,22 @@ class Dashboard: dashboardId: str +@dataclass_json +@dataclass +class RedshiftConnection: + secret_arn: str + namespace_id: str = None + workgroup: str = None + cluster_id: str = None + + @dataclass_json @dataclass class TestData: users: dict[str, User] envs: dict[str, Env] dashboards: dict[str, Dashboard] + redshift_connections: dict[str, RedshiftConnection] @pytest.fixture(scope='session', autouse=True) diff --git a/tests_new/integration_tests/modules/redshift_datasets/conftest.py b/tests_new/integration_tests/modules/redshift_datasets/conftest.py new file mode 100644 index 000000000..aa5b0c058 --- /dev/null +++ b/tests_new/integration_tests/modules/redshift_datasets/conftest.py @@ -0,0 +1,10 @@ +import logging + +import pytest +from integration_tests.core.stack.utils import check_stack_ready, wait_stack_delete_complete +from integration_tests.modules.redshift_datasets.connection_queries import ( + create_redshift_connection, + delete_redshift_connection, +) + +log = logging.getLogger(__name__) diff --git a/tests_new/integration_tests/modules/redshift_datasets/connection_queries.py b/tests_new/integration_tests/modules/redshift_datasets/connection_queries.py new file mode 100644 index 000000000..6cec9cb14 --- /dev/null +++ b/tests_new/integration_tests/modules/redshift_datasets/connection_queries.py @@ -0,0 +1,226 @@ +# TODO: This file will be replaced by using the SDK directly + + +def list_environment_redshift_connections(client, term='', environment_uri=None, group_uri=None, connection_type=None): + query = { + 'operationName': 'llistEnvironmentRedshiftConnections', + 'variables': { + 'filter': { + 'term': term, + 'environmentUri': environment_uri, + 'groupUri': group_uri, + 'connectionType': connection_type, + } + }, + 'query': """ + query listEnvironmentRedshiftConnections($filter: ConnectionFilter) { + listEnvironmentRedshiftConnections(filter: $filter) { + count + page + pages + hasNext + hasPrevious + nodes { + name + connectionUri + SamlGroupName + redshiftType + clusterId + nameSpaceId + workgroup + database + redshiftUser + secretArn + connectionType + } + } + } + """, + } + response = client.query(query=query) + return response.data.listEnvironmentRedshiftConnections + + +def list_redshift_connection_schemas(client, connection_uri): + query = { + 'operationName': 'listRedshiftConnectionSchemas', + 'variables': {'connectionUri': connection_uri}, + 'query': """ + query listRedshiftConnectionSchemas($connectionUri: String!) { + listRedshiftConnectionSchemas(connectionUri: $connectionUri) + } + """, + } + response = client.query(query=query) + return response.data.listRedshiftConnectionSchemas + + +def list_redshift_schema_tables(client, connection_uri, schema): + query = { + 'operationName': 'listRedshiftSchemaTables', + 'variables': {'connectionUri': connection_uri, 'schema': schema}, + 'query': """ + query listRedshiftSchemaTables($connectionUri: String!, $schema: String!) { + listRedshiftSchemaTables(connectionUri: $connectionUri, schema: $schema) { + name + type + } + } + """, + } + response = client.query(query=query) + return response.data.listRedshiftSchemaTables + + +def list_redshift_connection_group_permissions(client, connection_uri, term=''): + query = { + 'operationName': 'listConnectionGroupPermissions', + 'variables': {'connectionUri': connection_uri, 'filter': {'term': term}}, + 'query': """ + query listConnectionGroupPermissions( + $filter: GroupFilter + $connectionUri: String! + ) { + listConnectionGroupPermissions( + connectionUri: $connectionUri + filter: $filter + ) { + count + page + pages + hasNext + hasPrevious + nodes { + groupUri + permissions { + name + description + } + } + } + } + """, + } + response = client.query(query=query) + return response.data.listConnectionGroupPermissions + + +def list_redshift_connection_group_no_permissions(client, connection_uri, term=''): + query = { + 'operationName': 'listConnectionGroupNoPermissions', + 'variables': {'connectionUri': connection_uri, 'filter': {'term': term}}, + 'query': """ + query listConnectionGroupNoPermissions( + $filter: GroupFilter + $connectionUri: String! + ) { + listConnectionGroupNoPermissions( + connectionUri: $connectionUri + filter: $filter + ) + } + """, + } + response = client.query(query=query) + return response.data.listConnectionGroupNoPermissions + + +def create_redshift_connection( + client, + connection_name, + connection_type, + environment_uri, + group_uri, + redshift_type, + cluster_id, + namespace_id, + workgroup, + database, + redshift_user, + secret_arn, +): + query = { + 'operationName': 'createRedshiftConnection', + 'variables': { + 'input': { + 'connectionName': connection_name, + 'connectionType': connection_type, + 'environmentUri': environment_uri, + 'SamlGroupName': group_uri, + 'redshiftType': redshift_type, + 'clusterId': cluster_id, + 'nameSpaceId': namespace_id, + 'workgroup': workgroup, + 'database': database, + 'redshiftUser': redshift_user, + 'secretArn': secret_arn, + } + }, + 'query': """ + mutation createRedshiftConnection($input: CreateRedshiftConnectionInput) { + createRedshiftConnection(input: $input) { + connectionUri + connectionType + redshiftType + } + } + """, + } + response = client.query(query=query) + return response.data.createRedshiftConnection + + +def delete_redshift_connection(client, connection_uri): + query = { + 'operationName': 'deleteRedshiftConnection', + 'variables': {'connectionUri': connection_uri}, + 'query': """ + mutation deleteRedshiftConnection($connectionUri: String!) { + deleteRedshiftConnection(connectionUri: $connectionUri) + } + """, + } + response = client.query(query=query) + return response.data.deleteRedshiftConnection + + +def add_redshift_connection_group_permissions(client, connection_uri, group_uri, permissions): + query = { + 'operationName': 'addConnectionGroupPermission', + 'variables': {'connectionUri': connection_uri, 'groupUri': group_uri, 'permissions': permissions}, + 'query': """ + mutation addConnectionGroupPermission( + $connectionUri: String! + $groupUri: String! + $permissions: [String]! + ) { + addConnectionGroupPermission( + connectionUri: $connectionUri + groupUri: $groupUri + permissions: $permissions + ) + } + """, + } + response = client.query(query=query) + return response.data.addConnectionGroupPermission + + +def delete_redshift_connection_group_permissions(client, connection_uri, group_uri): + query = { + 'operationName': 'deleteConnectionGroupPermission', + 'variables': {'connectionUri': connection_uri, 'groupUri': group_uri}, + 'query': """ + mutation deleteConnectionGroupPermission( + $connectionUri: String! + $groupUri: String! + ) { + deleteConnectionGroupPermission( + connectionUri: $connectionUri + groupUri: $groupUri + ) + } + """, + } + response = client.query(query=query) + return response.data.deleteConnectionGroupPermission diff --git a/tests_new/integration_tests/modules/redshift_datasets/dataset_queries.py b/tests_new/integration_tests/modules/redshift_datasets/dataset_queries.py new file mode 100644 index 000000000..a96d90b2f --- /dev/null +++ b/tests_new/integration_tests/modules/redshift_datasets/dataset_queries.py @@ -0,0 +1,346 @@ +# TODO: This file will be replaced by using the SDK directly + + +def get_redshift_dataset(client, dataset_uri): + query = { + 'operationName': 'getRedshiftDataset', + 'variables': {'datasetUri': dataset_uri}, + 'query': """ + query getRedshiftDataset($datasetUri: String!) { + getRedshiftDataset(datasetUri: $datasetUri) { + datasetUri + owner + description + label + name + region + created + imported + userRoleForDataset + SamlAdminGroupName + AwsAccountId + tags + stewards + topics + confidentiality + autoApprovalEnabled + organization { + organizationUri + label + } + terms { + count + nodes { + __typename + ... on Term { + nodeUri + path + label + } + } + } + environment { + environmentUri + label + region + organization { + organizationUri + label + } + } + upvotes + connection { + connectionUri + label + redshiftType + clusterId + nameSpaceId + workgroup + redshiftUser + secretArn + database + } + schema + } + } + """, + } + response = client.query(query=query) + return response.data.getRedshiftDataset + + +def list_redshift_dataset_tables(client, dataset_uri, term=''): + query = { + 'operationName': 'listRedshiftDatasetTables', + 'variables': { + 'datasetUri': dataset_uri, + 'filter': {'term': term}, + }, + 'query': """ + query listRedshiftDatasetTables( + $datasetUri: String! + $filter: RedshiftDatasetTableFilter + ) { + listRedshiftDatasetTables(datasetUri: $datasetUri, filter: $filter) { + count + page + pages + hasNext + hasPrevious + nodes { + rsTableUri + datasetUri + name + label + created + description + } + } + } + """, + } + response = client.query(query=query) + return response.data.listRedshiftDatasetTables + + +def get_redshift_dataset_table(client, rs_table_uri): + query = { + 'operationName': 'getRedshiftDatasetTable', + 'variables': {'rsTableUri': rs_table_uri}, + 'query': """ + query getRedshiftDatasetTable($rsTableUri: String!) { + getRedshiftDatasetTable(rsTableUri: $rsTableUri) { + rsTableUri + name + label + created + description + tags + terms { + count + nodes { + __typename + ... on Term { + nodeUri + path + label + } + } + } + dataset { + owner + SamlAdminGroupName + datasetUri + name + label + userRoleForDataset + organization { + label + } + environment { + label + } + region + } + } + } + """, + } + response = client.query(query=query) + return response.data.getRedshiftDatasetTable + + +def get_redshift_dataset_table_columns(client, rs_table_uri, term=''): + query = { + 'operationName': 'getRedshiftDatasetTableColumns', + 'variables': { + 'rsTableUri': rs_table_uri, + 'filter': {'term': term}, + }, + 'query': """ + query getRedshiftDatasetTableColumns( + $rsTableUri: String! + $filter: RedshiftDatasetTableFilter + ) { + getRedshiftDatasetTableColumns(rsTableUri: $rsTableUri, filter: $filter) { + count + page + pages + hasNext + hasPrevious + nodes { + columnDefault + label + length + name + nullable + typeName + } + } + } + """, + } + response = client.query(query=query) + return response.data.getRedshiftDatasetTableColumns + + +def list_redshift_schema_dataset_tables(client, dataset_uri): + query = { + 'operationName': 'listRedshiftSchemaDatasetTables', + 'variables': {'datasetUri': dataset_uri}, + 'query': """ + query listRedshiftSchemaDatasetTables($datasetUri: String!) { + listRedshiftSchemaDatasetTables(datasetUri: $datasetUri) { + name + type + alreadyAdded + } + } + """, + } + response = client.query(query=query) + return response.data.listRedshiftSchemaDatasetTables + + +def import_redshift_dataset( + client, + label, + org_uri, + env_uri, + description, + tags, + owner, + topics, + group_uri, + confidentiality, + auto_approval_enabled, + connection_uri, + schema, + tables, +): + mutation = { + 'operationName': 'importRedshiftDataset', + 'variables': { + 'input': { + 'label': label, + 'organizationUri': org_uri, + 'environmentUri': env_uri, + 'description': description, + 'tags': tags, + 'owner': owner, + 'topics': topics, + 'SamlAdminGroupName': group_uri, + 'confidentiality': confidentiality, + 'autoApprovalEnabled': auto_approval_enabled, + 'connectionUri': connection_uri, + 'schema': schema, + 'tables': tables, + } + }, + 'query': """ + mutation importRedshiftDataset($input: ImportRedshiftDatasetInput) { + importRedshiftDataset(input: $input) { + datasetUri + label + userRoleForDataset + } + } + """, + } + response = client.query(mutation=mutation) + return response.data.importRedshiftDataset + + +def update_redshift_dataset(client, dataset_uri, description): + mutation = { + 'operationName': 'updateRedshiftDataset', + 'variables': { + 'datasetUri': dataset_uri, + 'input': {'description': description}, + }, + 'query': """ + mutation updateRedshiftDataset( + $datasetUri: String! + $input: ModifyRedshiftDatasetInput + ) { + updateRedshiftDataset(datasetUri: $datasetUri, input: $input) { + datasetUri + label + userRoleForDataset + } + } + """, + } + response = client.query(mutation=mutation) + return response.data.updateRedshiftDataset + + +def delete_redshift_dataset(client, dataset_uri): + mutation = { + 'operationName': 'deleteRedshiftDataset', + 'variables': {'datasetUri': dataset_uri}, + 'query': """ + mutation deleteRedshiftDataset($datasetUri: String!) { + deleteRedshiftDataset(datasetUri: $datasetUri) + } + """, + } + response = client.query(mutation=mutation) + return response.data.deleteRedshiftDataset + + +def add_redshift_dataset_tables(client, dataset_uri, tables): + mutation = { + 'operationName': 'addRedshiftDatasetTables', + 'variables': { + 'datasetUri': dataset_uri, + 'tables': tables, + }, + 'query': """ + mutation addRedshiftDatasetTables( + $datasetUri: String! + $tables: [String]! + ) { + addRedshiftDatasetTables(datasetUri: $datasetUri, tables: $tables) + } + """, + } + response = client.query(mutation=mutation) + return response.data.addRedshiftDatasetTables + + +def delete_redshift_dataset_table(client, rs_table_uri): + mutation = { + 'operationName': 'deleteRedshiftDatasetTable', + 'variables': {'rsTableUri': rs_table_uri}, + 'query': """ + mutation deleteRedshiftDatasetTable($rsTableUri: String!) { + deleteRedshiftDatasetTable(rsTableUri: $rsTableUri) + } + """, + } + response = client.query(mutation=mutation) + return response.data.deleteRedshiftDatasetTable + + +def update_redshift_dataset_table(client, rs_table_uri, description): + mutation = { + 'operationName': 'updateRedshiftDatasetTable', + 'variables': { + 'rsTableUri': rs_table_uri, + 'input': {'description': description}, + }, + 'query': """ + mutation updateRedshiftDatasetTable( + $rsTableUri: String! + $input: ModifyRedshiftDatasetInput + ) { + updateRedshiftDatasetTable(rsTableUri: $rsTableUri, input: $input) { + rsTableUri + label + } + } + """, + } + response = client.query(mutation=mutation) + return response.data.updateRedshiftDatasetTable diff --git a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py new file mode 100644 index 000000000..ea29a2342 --- /dev/null +++ b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py @@ -0,0 +1,133 @@ +import logging + +import pytest +from integration_tests.core.stack.utils import check_stack_ready, wait_stack_delete_complete +from integration_tests.conftest import RedshiftConnection +from integration_tests.modules.redshift_datasets.connection_queries import ( + create_redshift_connection, + delete_redshift_connection, +) + +log = logging.getLogger(__name__) + +REDSHIFT_DATABASE = 'dev' + + +def create_connection(client, env, group, name, conn_type, red_type, connection_data=RedshiftConnection): + connection = create_redshift_connection( + client=client, + connection_name=name, + connection_type=conn_type, + environment_uri=env.environmentUri, + group_uri=group, + redshift_type=red_type, + cluster_id=connection_data.cluster_id, + namespace_id=connection_data.namespace_id, + workgroup=connection_data.workgroup, + database=REDSHIFT_DATABASE, + redshift_user=None, + secret_arn=connection_data.secret_arn, + ) + check_stack_ready( + client=client, + env_uri=env.environmentUri, + stack_uri=env.stack.stackUri, + target_uri=env.environmentUri, + target_type='environment', + ) + return connection + + +""" +- Serverless namespace is deployed in session_env1 account +- Provisioned cluster is deployed in session_cross_acc_env_1 +""" + + +@pytest.fixture(scope='session') +def session_connection_serverless_admin(client1, group1, session_env1, testdata): + connection = None + try: + connection = create_connection( + client=client1, + name='connection_serverless_admin_session_env1', + conn_type='ADMIN', + env=session_env1, + group=group1, + red_type='serverless', + connection_data=testdata.redshift_connections['connection_serverless_admin_session_env1'], + ) + + yield connection + finally: + if connection: + delete_redshift_connection(client=client1, connection_uri=connection.connectionUri) + + +@pytest.fixture(scope='session') +def session_connection_serverless_data_user(client1, group1, session_env1, testdata): + connection = None + try: + connection = create_connection( + client=client1, + name='connection_serverless_data_user_session_env1', + conn_type='DATA_USER', + env=session_env1, + group=group1, + red_type='serverless', + connection_data=testdata.redshift_connections['connection_serverless_data_user_session_env1'], + ) + yield connection + finally: + if connection: + delete_redshift_connection(client=client1, connection_uri=connection.connectionUri) + + +@pytest.fixture(scope='session') +def session_connection_cluster_admin(client1, group1, session_cross_acc_env_1, testdata): + connection = None + try: + connection = create_connection( + client=client1, + name='connection_serverless_admin_session_env1', + conn_type='ADMIN', + env=session_cross_acc_env_1, + group=group1, + red_type='cluster', + connection_data=testdata.redshift_connections['connection_serverless_admin_session_env1'], + ) + yield connection + finally: + if connection: + delete_redshift_connection(client=client1, connection_uri=connection.connectionUri) + + +@pytest.fixture(scope='session') +def session_connection_cluster_data_user(client1, group1, session_cross_acc_env_1, testdata): + connection = None + try: + connection = create_connection( + client=client1, + name='connection_serverless_data_user_session_env1', + conn_type='DATA_USER', + env=session_cross_acc_env_1, + group=group1, + red_type='cluster', + connection_data=testdata.redshift_connections['connection_serverless_data_user_session_env1'], + ) + yield connection + finally: + if connection: + delete_redshift_connection(client=client1, connection_uri=connection.connectionUri) + + +@pytest.fixture(scope='session') +def session_redshift_dataset1_serverless( + client1, group1, session_env1, session_id, session_connection1_serverless_data_user +): + pass + + +@pytest.fixture(scope='session') +def session_redshift_dataset2_cluster(client1, group1, session_env1, session_id, session_connection2_cluster_data_user): + pass diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py new file mode 100644 index 000000000..c86fcb575 --- /dev/null +++ b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py @@ -0,0 +1,62 @@ +from assertpy import assert_that +import pytest + +from integration_tests.errors import GqlError +from integration_tests.modules.redshift_datasets.connection_queries import ( + create_redshift_connection, + delete_redshift_connection, +) +from integration_tests.modules.redshift_datasets.global_conftest import REDSHIFT_DATABASE + + +@pytest.mark.parametrize( + 'connection_fixture_name, connection_type, redshift_type', + [ + ('session_connection1_serverless_data_user', 'DATA_USER', 'serverless'), + ('session_connection2_cluster_data_user', 'DATA_USER', 'cluster'), + ('session_connection3_serverless_admin', 'ADMIN', 'serverless'), + ('session_connection4_cluster_admin', 'ADMIN', 'cluster'), + ], +) +def test_create_connection(client1, connection_fixture_name, connection_type, redshift_type, request): + connection = request.getfixturevalue(connection_fixture_name) + assert_that(connection.connectionUri).is_not_none() + assert_that(connection.connectionType).is_equal_to(connection_type) + assert_that(connection.redshiftType).is_equal_to(redshift_type) + + +def test_create_serverless_connection_namespace_does_not_exist(client1, group1, session_env1, testdata): + connection_data = testdata.redshift_connections.get('connection_1') + ERROR_NAMESPACE_ID = 'doesNotExistNamespace' + assert_that(create_redshift_connection).raises(GqlError).when_called_with( + client=client1, + connection_name='errorConnection', + environment_uri=session_env1.environmentUri, + group_uri=group1, + redshift_type='serverless', + namespace_id=ERROR_NAMESPACE_ID, + workgroup=connection_data.workgroup, + database=REDSHIFT_DATABASE, + redshift_user=None, + secret_arn=connection_data.secret_arn, + ).contains('Redshift namespaceId', ERROR_NAMESPACE_ID, 'not exist') + + +def test_create_serverless_connection_workgroup_not_found(): + pass + + +def test_create_cluster_connection_cluster_not_found(): + pass + + +def test_create_cluster_connection_cluster_not_encrypted(): + pass + + +def test_create_connection_database_not_found(): + pass + + +def test_create_connection_unauthorized(): + pass diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_dataset.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_dataset.py new file mode 100644 index 000000000..e69de29bb From d658890729980fc7eba631e0e4700b9d8555b548 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Thu, 10 Oct 2024 14:35:16 +0200 Subject: [PATCH 2/7] Added redshift-connection tests --- .../modules/redshift_datasets/aws/redshift.py | 9 +- .../db/redshift_connection_repositories.py | 2 - .../redshift_datasets/connection_queries.py | 18 +- .../redshift_datasets/global_conftest.py | 56 +-- .../test_redshift_connection.py | 330 +++++++++++++++++- .../test_redshift_dataset.py | 0 6 files changed, 364 insertions(+), 51 deletions(-) delete mode 100644 tests_new/integration_tests/modules/redshift_datasets/test_redshift_dataset.py diff --git a/backend/dataall/modules/redshift_datasets/aws/redshift.py b/backend/dataall/modules/redshift_datasets/aws/redshift.py index 7a6c808cf..f523fe477 100644 --- a/backend/dataall/modules/redshift_datasets/aws/redshift.py +++ b/backend/dataall/modules/redshift_datasets/aws/redshift.py @@ -14,7 +14,14 @@ def __init__(self, account_id: str, region: str) -> None: def describe_cluster(self, clusterId: str): log.info(f'Describing cluster {clusterId=}') - return self.client.describe_clusters(ClusterIdentifier=clusterId)['Clusters'][0] + try: + return self.client.describe_clusters(ClusterIdentifier=clusterId)['Clusters'][0] + except ClientError as e: + if e.response['Error']['Code'] == 'ClusterNotFound': + log.error(f'Redshift cluster {clusterId} does not exist') + return None + else: + raise e def get_cluster_namespaceId(self, clusterId: str): log.info(f'Describing cluster {clusterId=}') diff --git a/backend/dataall/modules/redshift_datasets/db/redshift_connection_repositories.py b/backend/dataall/modules/redshift_datasets/db/redshift_connection_repositories.py index 110d6e0ef..37751d7f2 100644 --- a/backend/dataall/modules/redshift_datasets/db/redshift_connection_repositories.py +++ b/backend/dataall/modules/redshift_datasets/db/redshift_connection_repositories.py @@ -128,8 +128,6 @@ def _query_redshift_connection_group_permissions(session, connection_uri, permis .group_by(ResourcePolicy.principalId) ) - if filter and filter.get('groupUri'): - query = query.filter(ResourcePolicy.principalId == filter.get('groupUri')) if filter and filter.get('term'): query = query.filter( ResourcePolicy.principalId.ilike(filter.get('term') + '%%'), diff --git a/tests_new/integration_tests/modules/redshift_datasets/connection_queries.py b/tests_new/integration_tests/modules/redshift_datasets/connection_queries.py index 6cec9cb14..5b5e3ea0c 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/connection_queries.py +++ b/tests_new/integration_tests/modules/redshift_datasets/connection_queries.py @@ -3,7 +3,7 @@ def list_environment_redshift_connections(client, term='', environment_uri=None, group_uri=None, connection_type=None): query = { - 'operationName': 'llistEnvironmentRedshiftConnections', + 'operationName': 'listEnvironmentRedshiftConnections', 'variables': { 'filter': { 'term': term, @@ -72,10 +72,10 @@ def list_redshift_schema_tables(client, connection_uri, schema): return response.data.listRedshiftSchemaTables -def list_redshift_connection_group_permissions(client, connection_uri, term=''): +def list_redshift_connection_group_permissions(client, connection_uri, filter={}): query = { 'operationName': 'listConnectionGroupPermissions', - 'variables': {'connectionUri': connection_uri, 'filter': {'term': term}}, + 'variables': {'connectionUri': connection_uri, 'filter': filter}, 'query': """ query listConnectionGroupPermissions( $filter: GroupFilter @@ -128,16 +128,16 @@ def list_redshift_connection_group_no_permissions(client, connection_uri, term=' def create_redshift_connection( client, connection_name, - connection_type, environment_uri, group_uri, redshift_type, - cluster_id, - namespace_id, - workgroup, database, - redshift_user, - secret_arn, + connection_type, + cluster_id=None, + namespace_id=None, + workgroup=None, + redshift_user=None, + secret_arn=None, ): query = { 'operationName': 'createRedshiftConnection', diff --git a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py index ea29a2342..19082161c 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py +++ b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py @@ -6,6 +6,8 @@ from integration_tests.modules.redshift_datasets.connection_queries import ( create_redshift_connection, delete_redshift_connection, + add_redshift_connection_group_permissions, + delete_redshift_connection_group_permissions, ) log = logging.getLogger(__name__) @@ -64,6 +66,24 @@ def session_connection_serverless_admin(client1, group1, session_env1, testdata) delete_redshift_connection(client=client1, connection_uri=connection.connectionUri) +@pytest.fixture(scope='session') +def session_connection_serverless_admin_group_with_permissions(client1, group2, session_connection_serverless_admin): + permissions = None + try: + permissions = add_redshift_connection_group_permissions( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + group_uri=group2, + permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], + ) + yield group2 + finally: + if permissions: + delete_redshift_connection_group_permissions( + client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group2 + ) + + @pytest.fixture(scope='session') def session_connection_serverless_data_user(client1, group1, session_env1, testdata): connection = None @@ -84,50 +104,38 @@ def session_connection_serverless_data_user(client1, group1, session_env1, testd @pytest.fixture(scope='session') -def session_connection_cluster_admin(client1, group1, session_cross_acc_env_1, testdata): +def session_connection_cluster_admin(client5, group5, session_cross_acc_env_1, testdata): connection = None try: connection = create_connection( - client=client1, - name='connection_serverless_admin_session_env1', + client=client5, + name='connection_cluster_admin_session_cross_acc_env_1', conn_type='ADMIN', env=session_cross_acc_env_1, - group=group1, + group=group5, red_type='cluster', - connection_data=testdata.redshift_connections['connection_serverless_admin_session_env1'], + connection_data=testdata.redshift_connections['connection_cluster_admin_session_cross_acc_env_1'], ) yield connection finally: if connection: - delete_redshift_connection(client=client1, connection_uri=connection.connectionUri) + delete_redshift_connection(client=client5, connection_uri=connection.connectionUri) @pytest.fixture(scope='session') -def session_connection_cluster_data_user(client1, group1, session_cross_acc_env_1, testdata): +def session_connection_cluster_data_user(client5, group5, session_cross_acc_env_1, testdata): connection = None try: connection = create_connection( - client=client1, - name='connection_serverless_data_user_session_env1', + client=client5, + name='connection_cluster_data_user_session_cross_acc_env_1', conn_type='DATA_USER', env=session_cross_acc_env_1, - group=group1, + group=group5, red_type='cluster', - connection_data=testdata.redshift_connections['connection_serverless_data_user_session_env1'], + connection_data=testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'], ) yield connection finally: if connection: - delete_redshift_connection(client=client1, connection_uri=connection.connectionUri) - - -@pytest.fixture(scope='session') -def session_redshift_dataset1_serverless( - client1, group1, session_env1, session_id, session_connection1_serverless_data_user -): - pass - - -@pytest.fixture(scope='session') -def session_redshift_dataset2_cluster(client1, group1, session_env1, session_id, session_connection2_cluster_data_user): - pass + delete_redshift_connection(client=client5, connection_uri=connection.connectionUri) diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py index c86fcb575..ac6e8d7cd 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py +++ b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py @@ -5,6 +5,13 @@ from integration_tests.modules.redshift_datasets.connection_queries import ( create_redshift_connection, delete_redshift_connection, + add_redshift_connection_group_permissions, + delete_redshift_connection_group_permissions, + list_environment_redshift_connections, + list_redshift_connection_schemas, + list_redshift_schema_tables, + list_redshift_connection_group_permissions, + list_redshift_connection_group_no_permissions, ) from integration_tests.modules.redshift_datasets.global_conftest import REDSHIFT_DATABASE @@ -12,13 +19,13 @@ @pytest.mark.parametrize( 'connection_fixture_name, connection_type, redshift_type', [ - ('session_connection1_serverless_data_user', 'DATA_USER', 'serverless'), - ('session_connection2_cluster_data_user', 'DATA_USER', 'cluster'), - ('session_connection3_serverless_admin', 'ADMIN', 'serverless'), - ('session_connection4_cluster_admin', 'ADMIN', 'cluster'), + ('session_connection_serverless_data_user', 'DATA_USER', 'serverless'), + ('session_connection_cluster_data_user', 'DATA_USER', 'cluster'), + ('session_connection_serverless_admin', 'ADMIN', 'serverless'), + ('session_connection_cluster_admin', 'ADMIN', 'cluster'), ], ) -def test_create_connection(client1, connection_fixture_name, connection_type, redshift_type, request): +def test_create_connection(connection_fixture_name, connection_type, redshift_type, request): connection = request.getfixturevalue(connection_fixture_name) assert_that(connection.connectionUri).is_not_none() assert_that(connection.connectionType).is_equal_to(connection_type) @@ -26,14 +33,15 @@ def test_create_connection(client1, connection_fixture_name, connection_type, re def test_create_serverless_connection_namespace_does_not_exist(client1, group1, session_env1, testdata): - connection_data = testdata.redshift_connections.get('connection_1') - ERROR_NAMESPACE_ID = 'doesNotExistNamespace' + connection_data = testdata.redshift_connections['connection_serverless_data_user_session_env1'] + ERROR_NAMESPACE_ID = 'doesnotexist' assert_that(create_redshift_connection).raises(GqlError).when_called_with( client=client1, connection_name='errorConnection', environment_uri=session_env1.environmentUri, group_uri=group1, redshift_type='serverless', + connection_type='DATA_USER', namespace_id=ERROR_NAMESPACE_ID, workgroup=connection_data.workgroup, database=REDSHIFT_DATABASE, @@ -42,21 +50,313 @@ def test_create_serverless_connection_namespace_does_not_exist(client1, group1, ).contains('Redshift namespaceId', ERROR_NAMESPACE_ID, 'not exist') -def test_create_serverless_connection_workgroup_not_found(): - pass +def test_create_serverless_connection_workgroup_not_found(client1, group1, session_env1, testdata): + connection_data = testdata.redshift_connections['connection_serverless_data_user_session_env1'] + ERROR_WORKGROUP = 'doesnotexist' + assert_that(create_redshift_connection).raises(GqlError).when_called_with( + client=client1, + connection_name='errorConnection', + environment_uri=session_env1.environmentUri, + group_uri=group1, + redshift_type='serverless', + connection_type='DATA_USER', + namespace_id=connection_data.namespace_id, + workgroup=ERROR_WORKGROUP, + database=REDSHIFT_DATABASE, + redshift_user=None, + secret_arn=connection_data.secret_arn, + ).contains('Redshift workgroup', ERROR_WORKGROUP, 'not exist') -def test_create_cluster_connection_cluster_not_found(): - pass +def test_create_cluster_connection_cluster_not_found(client5, group5, session_cross_acc_env_1, testdata): + connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'] + ERROR_CLUSTER_ID = 'doesnotexist' + assert_that(create_redshift_connection).raises(GqlError).when_called_with( + client=client5, + connection_name='errorConnection', + environment_uri=session_cross_acc_env_1.environmentUri, + group_uri=group5, + redshift_type='cluster', + connection_type='DATA_USER', + cluster_id=ERROR_CLUSTER_ID, + database=REDSHIFT_DATABASE, + redshift_user=None, + secret_arn=connection_data.secret_arn, + ).contains('Redshift cluster', ERROR_CLUSTER_ID, 'not exist') def test_create_cluster_connection_cluster_not_encrypted(): + # TODO: we need to decide if we want to create an extra cluster that is not encrypted pass -def test_create_connection_database_not_found(): - pass +def test_create_connection_database_not_found(client5, group5, session_cross_acc_env_1, testdata): + connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'] + ERROR_DATABASE = 'doesnotexist' + assert_that(create_redshift_connection).raises(GqlError).when_called_with( + client=client5, + connection_name='errorConnection', + environment_uri=session_cross_acc_env_1.environmentUri, + group_uri=group5, + redshift_type='cluster', + connection_type='DATA_USER', + cluster_id=connection_data.cluster_id, + database=ERROR_DATABASE, + redshift_user=None, + secret_arn=connection_data.secret_arn, + ).contains('Redshift database', ERROR_DATABASE, 'not exist') -def test_create_connection_unauthorized(): - pass +def test_create_connection_unauthorized(client1, group1, session_cross_acc_env_1, testdata): + connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'] + assert_that(create_redshift_connection).raises(GqlError).when_called_with( + client=client1, + connection_name='errorConnection', + environment_uri=session_cross_acc_env_1.environmentUri, + group_uri=group1, + redshift_type='cluster', + connection_type='DATA_USER', + cluster_id=connection_data.cluster_id, + database=REDSHIFT_DATABASE, + redshift_user=None, + secret_arn=connection_data.secret_arn, + ).contains('UnauthorizedOperation', 'CREATE_REDSHIFT_CONNECTION', session_cross_acc_env_1.environmentUri) + + +def test_delete_connection(client5, group5, session_cross_acc_env_1, testdata): + connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'] + connection = create_redshift_connection( + client=client5, + connection_name='errorConnection', + environment_uri=session_cross_acc_env_1.environmentUri, + group_uri=group5, + redshift_type='cluster', + connection_type='DATA_USER', + cluster_id=connection_data.cluster_id, + database=REDSHIFT_DATABASE, + redshift_user=None, + secret_arn=connection_data.secret_arn, + ) + response = delete_redshift_connection( + client=client5, + connection_uri=connection.connectionUri, + ) + assert_that(response).is_true() + + +def test_delete_connection_unauthorized(client2, session_connection_serverless_admin): + assert_that(delete_redshift_connection).raises(GqlError).when_called_with( + client=client2, + connection_uri=session_connection_serverless_admin.connectionUri, + ).contains('UnauthorizedOperation', 'DELETE_REDSHIFT_CONNECTION', session_connection_serverless_admin.connectionUri) + + +def test_add_connection_group_permissions(client1, group2, session_connection_serverless_admin_group_with_permissions): + assert_that(session_connection_serverless_admin_group_with_permissions).is_equal_to(group2) + + +def test_add_connection_group_permissions_unauthorized(client2, group3, session_connection_serverless_admin): + assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( + client=client2, + connection_uri=session_connection_serverless_admin.connectionUri, + group_uri=group3, + permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], + ).contains( + 'UnauthorizedOperation', + 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + session_connection_serverless_admin.connectionUri, + ) + + +def test_add_connection_group_permissions_invalid_connection_type( + client1, group3, session_connection_serverless_data_user +): + assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( + client=client1, + connection_uri=session_connection_serverless_data_user.connectionUri, + group_uri=group3, + permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], + ).contains('InvalidInput', 'ConnectionType', session_connection_serverless_data_user.connectionType) + + +def test_add_connection_group_permissions_invalid_permissions(client1, group3, session_connection_serverless_admin): + assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + group_uri=group3, + permissions=['INVALID_PERMISSION'], + ).contains('InvalidInput', 'INVALID_PERMISSION', 'Permissions') + + +def test_delete_connection_group_permissions(client1, group3, session_connection_serverless_admin): + response = add_redshift_connection_group_permissions( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + group_uri=group3, + permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], + ) + assert_that(response).is_true() + response = delete_redshift_connection_group_permissions( + client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group3 + ) + assert_that(response).is_true() + + +def test_delete_connection_group_permissions_unauthorized(client2, group3, session_connection_serverless_admin): + assert_that(delete_redshift_connection_group_permissions).raises(GqlError).when_called_with( + client=client2, + connection_uri=session_connection_serverless_admin.connectionUri, + group_uri=group3, + ).contains( + 'UnauthorizedOperation', + 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + session_connection_serverless_admin.connectionUri, + ) + + +def test_delete_connection_group_permissions_invalid_connection_type( + client1, group3, session_connection_serverless_data_user +): + assert_that(delete_redshift_connection_group_permissions).raises(GqlError).when_called_with( + client=client1, + connection_uri=session_connection_serverless_data_user.connectionUri, + group_uri=group3, + ).contains('InvalidInput', 'ConnectionType', session_connection_serverless_data_user.connectionType) + + +def test_delete_connection_group_permissions_invalid_group(client1, group1, session_connection_serverless_admin): + assert_that(delete_redshift_connection_group_permissions).raises(GqlError).when_called_with( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + group_uri=group1, + ).contains('InvalidInput', 'Team', group1, 'EXCEPT the connection owners') + + +def test_list_redshift_environment_connections( + client1, group1, session_env1, session_connection_serverless_admin, session_connection_serverless_data_user +): + response = list_environment_redshift_connections( + client=client1, + environment_uri=session_env1.environmentUri, + group_uri=group1, + ) + assert_that(response.count).is_equal_to(2) + assert_that(response.nodes).extracting('connectionUri').contains( + session_connection_serverless_admin.connectionUri, session_connection_serverless_data_user.connectionUri + ) + response = list_environment_redshift_connections( + client=client1, + environment_uri=session_env1.environmentUri, + group_uri=group1, + connection_type='DATA_USER', + ) + assert_that(response.count).is_equal_to(1) + assert_that(response.nodes).extracting('connectionUri').contains( + session_connection_serverless_data_user.connectionUri + ) + assert_that(response.nodes).extracting('connectionUri').does_not_contain( + session_connection_serverless_admin.connectionUri + ) + + +def test_list_redshift_environment_connections_unauthorized(client2, group1, session_env1): + assert_that(list_environment_redshift_connections).raises(GqlError).when_called_with( + client=client2, + environment_uri=session_env1.environmentUri, + group_uri=group1, + ).contains( + 'UnauthorizedOperation', + 'LIST_ENVIRONMENT_REDSHIFT_CONNECTIONS', + session_env1.environmentUri, + ) + + +def test_list_redshift_connection_schemas(client1, session_connection_serverless_admin): + response = list_redshift_connection_schemas( + client=client1, connection_uri=session_connection_serverless_admin.connectionUri + ) + assert_that(len(response)).is_greater_than_or_equal_to(1) + assert_that(response).contains('public') + + +def test_list_redshift_connection_schemas_unauthorized(client2, session_connection_serverless_admin): + assert_that(list_redshift_connection_schemas).raises(GqlError).when_called_with( + client=client2, connection_uri=session_connection_serverless_admin.connectionUri + ).contains( + 'UnauthorizedOperation', + 'GET_REDSHIFT_CONNECTION', + session_connection_serverless_admin.connectionUri, + ) + + +def test_list_redshift_schema_tables(client1, session_connection_serverless_admin): + response = list_redshift_schema_tables( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + schema='public', + ) + assert_that(len(response)).is_greater_than_or_equal_to(1) + assert_that(response[0]).contains_key('name', 'type', 'alreadyAdded') + + +def test_list_redshift_schema_tables_unauthorized(client2, session_connection_serverless_admin): + assert_that(list_redshift_schema_tables).raises(GqlError).when_called_with( + client=client2, connection_uri=session_connection_serverless_admin.connectionUri, schema='public' + ).contains( + 'UnauthorizedOperation', + 'GET_REDSHIFT_CONNECTION', + session_connection_serverless_admin.connectionUri, + ) + + +def test_list_redshift_connection_group_permissions( + client1, group1, session_connection_serverless_admin, session_connection_serverless_admin_group_with_permissions +): + response = list_redshift_connection_group_permissions( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + ) + assert_that(response.count).is_equal_to(2) + assert_that(response.nodes).extracting('groupUri').contains( + group1, session_connection_serverless_admin_group_with_permissions + ) + response = list_redshift_connection_group_permissions( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + filter={'term': session_connection_serverless_admin_group_with_permissions}, + ) + assert_that(response.count).is_equal_to(1) + assert_that(response.nodes).extracting('groupUri').contains( + session_connection_serverless_admin_group_with_permissions + ) + + +def test_list_redshift_connection_group_permissions_unauthorized(client2, session_connection_serverless_admin): + assert_that(list_redshift_connection_group_permissions).raises(GqlError).when_called_with( + client=client2, connection_uri=session_connection_serverless_admin.connectionUri + ).contains( + 'UnauthorizedOperation', + 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + session_connection_serverless_admin.connectionUri, + ) + + +def test_list_redshift_connection_group_no_permissions( + client1, group1, session_connection_serverless_admin, session_connection_serverless_admin_group_with_permissions +): + response = list_redshift_connection_group_permissions( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + ) + assert_that(response).does_not_contain(session_connection_serverless_admin_group_with_permissions, group1) + assert_that(response).is_not_none() + + +def test_list_redshift_connection_group_no_permissions_unauthorized(client2, session_connection_serverless_admin): + assert_that(list_redshift_connection_group_no_permissions).raises(GqlError).when_called_with( + client=client2, connection_uri=session_connection_serverless_admin.connectionUri + ).contains( + 'UnauthorizedOperation', + 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + session_connection_serverless_admin.connectionUri, + ) diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_dataset.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_dataset.py deleted file mode 100644 index e69de29bb..000000000 From 6b646137cee634855229fd4202706b6970980338 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Fri, 11 Oct 2024 10:18:07 +0200 Subject: [PATCH 3/7] Fix Nones in redshift cdk policy and add update stack on connection delete --- .../redshift_datasets/cdk/pivot_role_redshift_policy.py | 6 ++---- .../services/redshift_connection_service.py | 1 + 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py b/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py index 8eb7c7712..66c52b89c 100644 --- a/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py +++ b/backend/dataall/modules/redshift_datasets/cdk/pivot_role_redshift_policy.py @@ -76,12 +76,10 @@ def get_statements(self): cluster_arns = [ f'arn:aws:redshift:{self.region}:{self.account}:cluster:{conn.clusterId}' for conn in connections - if conn.clusterId != '' + if conn.clusterId ] workgroup_arns = [ - rs_client.get_workgroup_arn(workgroup_name=conn.workgroup) - for conn in connections - if conn.workgroup != '' + rs_client.get_workgroup_arn(workgroup_name=conn.workgroup) for conn in connections if conn.workgroup ] additional_statements.extend( split_policy_with_resources_in_statements( diff --git a/backend/dataall/modules/redshift_datasets/services/redshift_connection_service.py b/backend/dataall/modules/redshift_datasets/services/redshift_connection_service.py index 3f0578049..80e4bbade 100644 --- a/backend/dataall/modules/redshift_datasets/services/redshift_connection_service.py +++ b/backend/dataall/modules/redshift_datasets/services/redshift_connection_service.py @@ -113,6 +113,7 @@ def delete_redshift_connection(uri) -> bool: ) session.delete(connection) session.commit() + StackService.deploy_stack(targetUri=connection.environmentUri) return True @staticmethod From eaa2d534d78eebd54430226392b5d8a320d8f074 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Fri, 11 Oct 2024 13:25:08 +0200 Subject: [PATCH 4/7] Fix assertion in test_redshift_schema_tables + clean PR --- tests_new/integration_tests/conftest.py | 1 - .../modules/redshift_datasets/conftest.py | 10 ---------- .../modules/redshift_datasets/global_conftest.py | 2 +- .../redshift_datasets/test_redshift_connection.py | 2 +- 4 files changed, 2 insertions(+), 13 deletions(-) delete mode 100644 tests_new/integration_tests/modules/redshift_datasets/conftest.py diff --git a/tests_new/integration_tests/conftest.py b/tests_new/integration_tests/conftest.py index ad9695b75..fa7f78e8e 100644 --- a/tests_new/integration_tests/conftest.py +++ b/tests_new/integration_tests/conftest.py @@ -3,7 +3,6 @@ import os import re import sys -from typing import Optional from dataclasses import dataclass import pytest diff --git a/tests_new/integration_tests/modules/redshift_datasets/conftest.py b/tests_new/integration_tests/modules/redshift_datasets/conftest.py deleted file mode 100644 index aa5b0c058..000000000 --- a/tests_new/integration_tests/modules/redshift_datasets/conftest.py +++ /dev/null @@ -1,10 +0,0 @@ -import logging - -import pytest -from integration_tests.core.stack.utils import check_stack_ready, wait_stack_delete_complete -from integration_tests.modules.redshift_datasets.connection_queries import ( - create_redshift_connection, - delete_redshift_connection, -) - -log = logging.getLogger(__name__) diff --git a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py index 19082161c..2efcdf7bb 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py +++ b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py @@ -1,7 +1,7 @@ import logging import pytest -from integration_tests.core.stack.utils import check_stack_ready, wait_stack_delete_complete +from integration_tests.core.stack.utils import check_stack_ready from integration_tests.conftest import RedshiftConnection from integration_tests.modules.redshift_datasets.connection_queries import ( create_redshift_connection, diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py index ac6e8d7cd..c5b4bb664 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py +++ b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py @@ -296,7 +296,7 @@ def test_list_redshift_schema_tables(client1, session_connection_serverless_admi schema='public', ) assert_that(len(response)).is_greater_than_or_equal_to(1) - assert_that(response[0]).contains_key('name', 'type', 'alreadyAdded') + assert_that(response[0]).contains_key('name', 'type') def test_list_redshift_schema_tables_unauthorized(client2, session_connection_serverless_admin): From e87979493b51991749d591eaff75c2a9aa7ec77c Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 14 Oct 2024 16:11:26 +0200 Subject: [PATCH 5/7] PR review comments --- tests_new/integration_tests/README.md | 13 ++++---- .../redshift_datasets/global_conftest.py | 1 + .../test_redshift_connection.py | 32 +++++++++---------- 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/tests_new/integration_tests/README.md b/tests_new/integration_tests/README.md index d9579345b..93e3517a6 100644 --- a/tests_new/integration_tests/README.md +++ b/tests_new/integration_tests/README.md @@ -97,7 +97,7 @@ Currently **we support only Cognito based deployments** but support for any IdP "cluster_id": "...", "secret_arn": "..." }, - "connection_cluster_data_admin_session_cross_acc_env_1": { + "connection_cluster_data_user_session_cross_acc_env_1": { "cluster_id": "...", "secret_arn": "..." } @@ -107,12 +107,13 @@ Currently **we support only Cognito based deployments** but support for any IdP - The pipeline will create the users/groups - For Redshift testing we require some pre-existing infrastructure: - - One Redshift serverless namespace+workgroup and one Redshift provisioned cluster - both MUST be encrypted - - Both must host the default `dev` database with the `public` schema. - - For each we need to ensure that the admin credentials are stored in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True} - - For each we need to create a Redshift user (see SQL commands below) and store the credentials in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True} + - One Redshift serverless namespace+workgroup deployed in `session_env1` and one Redshift provisioned cluster in `session_cross_acc_env_1` + - The provisioned cluster MUST be encrypted and use RA3 cluster type (Check the [docs](https://docs.aws.amazon.com/redshift/latest/dg/datashare-overview.html) for other data sharing limitations) + - Both clusters must host the default `dev` database with the `public` schema. + - For each we need to ensure that the admin credentials are stored in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True}. If you are going to use the Redshift Query Editor, then you will also need the tag {key:Redshift, value:any} + - For each we need to create a Redshift user (see SQL commands below) and store the credentials in Secrets Manager. The secrets MUST be tagged with the tag {key:dataall, value:True}. If you are going to use the Redshift Query Editor, then you will also need the tag {key:Redshift, value:any} - For each we need to create a set of tables using the commands below - - For each we need to create a Redshift role () + - For each we need to create a Redshift role as in the commands below Create User and grant basic permissions using admin connection ```sql diff --git a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py index 2efcdf7bb..ce58f3929 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py +++ b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py @@ -13,6 +13,7 @@ log = logging.getLogger(__name__) REDSHIFT_DATABASE = 'dev' +REDSHIFT_SCHEMA = 'public' def create_connection(client, env, group, name, conn_type, red_type, connection_data=RedshiftConnection): diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py index c5b4bb664..38597ca32 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py +++ b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py @@ -13,7 +13,7 @@ list_redshift_connection_group_permissions, list_redshift_connection_group_no_permissions, ) -from integration_tests.modules.redshift_datasets.global_conftest import REDSHIFT_DATABASE +from integration_tests.modules.redshift_datasets.global_conftest import REDSHIFT_DATABASE, REDSHIFT_SCHEMA @pytest.mark.parametrize( @@ -34,7 +34,7 @@ def test_create_connection(connection_fixture_name, connection_type, redshift_ty def test_create_serverless_connection_namespace_does_not_exist(client1, group1, session_env1, testdata): connection_data = testdata.redshift_connections['connection_serverless_data_user_session_env1'] - ERROR_NAMESPACE_ID = 'doesnotexist' + error_namespace_id = 'doesnotexist' assert_that(create_redshift_connection).raises(GqlError).when_called_with( client=client1, connection_name='errorConnection', @@ -42,17 +42,17 @@ def test_create_serverless_connection_namespace_does_not_exist(client1, group1, group_uri=group1, redshift_type='serverless', connection_type='DATA_USER', - namespace_id=ERROR_NAMESPACE_ID, + namespace_id=error_namespace_id, workgroup=connection_data.workgroup, database=REDSHIFT_DATABASE, redshift_user=None, secret_arn=connection_data.secret_arn, - ).contains('Redshift namespaceId', ERROR_NAMESPACE_ID, 'not exist') + ).contains('Redshift namespaceId', error_namespace_id, 'not exist') def test_create_serverless_connection_workgroup_not_found(client1, group1, session_env1, testdata): connection_data = testdata.redshift_connections['connection_serverless_data_user_session_env1'] - ERROR_WORKGROUP = 'doesnotexist' + error_workgroup = 'doesnotexist' assert_that(create_redshift_connection).raises(GqlError).when_called_with( client=client1, connection_name='errorConnection', @@ -61,16 +61,16 @@ def test_create_serverless_connection_workgroup_not_found(client1, group1, sessi redshift_type='serverless', connection_type='DATA_USER', namespace_id=connection_data.namespace_id, - workgroup=ERROR_WORKGROUP, + workgroup=error_workgroup, database=REDSHIFT_DATABASE, redshift_user=None, secret_arn=connection_data.secret_arn, - ).contains('Redshift workgroup', ERROR_WORKGROUP, 'not exist') + ).contains('Redshift workgroup', error_workgroup, 'not exist') def test_create_cluster_connection_cluster_not_found(client5, group5, session_cross_acc_env_1, testdata): connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'] - ERROR_CLUSTER_ID = 'doesnotexist' + error_cluster_id = 'doesnotexist' assert_that(create_redshift_connection).raises(GqlError).when_called_with( client=client5, connection_name='errorConnection', @@ -78,11 +78,11 @@ def test_create_cluster_connection_cluster_not_found(client5, group5, session_cr group_uri=group5, redshift_type='cluster', connection_type='DATA_USER', - cluster_id=ERROR_CLUSTER_ID, + cluster_id=error_cluster_id, database=REDSHIFT_DATABASE, redshift_user=None, secret_arn=connection_data.secret_arn, - ).contains('Redshift cluster', ERROR_CLUSTER_ID, 'not exist') + ).contains('Redshift cluster', error_cluster_id, 'not exist') def test_create_cluster_connection_cluster_not_encrypted(): @@ -92,7 +92,7 @@ def test_create_cluster_connection_cluster_not_encrypted(): def test_create_connection_database_not_found(client5, group5, session_cross_acc_env_1, testdata): connection_data = testdata.redshift_connections['connection_cluster_data_user_session_cross_acc_env_1'] - ERROR_DATABASE = 'doesnotexist' + error_database = 'doesnotexist' assert_that(create_redshift_connection).raises(GqlError).when_called_with( client=client5, connection_name='errorConnection', @@ -101,10 +101,10 @@ def test_create_connection_database_not_found(client5, group5, session_cross_acc redshift_type='cluster', connection_type='DATA_USER', cluster_id=connection_data.cluster_id, - database=ERROR_DATABASE, + database=error_database, redshift_user=None, secret_arn=connection_data.secret_arn, - ).contains('Redshift database', ERROR_DATABASE, 'not exist') + ).contains('Redshift database', error_database, 'not exist') def test_create_connection_unauthorized(client1, group1, session_cross_acc_env_1, testdata): @@ -276,7 +276,7 @@ def test_list_redshift_connection_schemas(client1, session_connection_serverless client=client1, connection_uri=session_connection_serverless_admin.connectionUri ) assert_that(len(response)).is_greater_than_or_equal_to(1) - assert_that(response).contains('public') + assert_that(response).contains(REDSHIFT_SCHEMA) def test_list_redshift_connection_schemas_unauthorized(client2, session_connection_serverless_admin): @@ -293,7 +293,7 @@ def test_list_redshift_schema_tables(client1, session_connection_serverless_admi response = list_redshift_schema_tables( client=client1, connection_uri=session_connection_serverless_admin.connectionUri, - schema='public', + schema=REDSHIFT_SCHEMA, ) assert_that(len(response)).is_greater_than_or_equal_to(1) assert_that(response[0]).contains_key('name', 'type') @@ -301,7 +301,7 @@ def test_list_redshift_schema_tables(client1, session_connection_serverless_admi def test_list_redshift_schema_tables_unauthorized(client2, session_connection_serverless_admin): assert_that(list_redshift_schema_tables).raises(GqlError).when_called_with( - client=client2, connection_uri=session_connection_serverless_admin.connectionUri, schema='public' + client=client2, connection_uri=session_connection_serverless_admin.connectionUri, schema=REDSHIFT_SCHEMA ).contains( 'UnauthorizedOperation', 'GET_REDSHIFT_CONNECTION', From b2a2c765c5e121cc48ffc6aa83fe2ec839ffaad9 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 14 Oct 2024 16:50:36 +0200 Subject: [PATCH 6/7] PR review comments --- .../core/environment/global_conftest.py | 3 +- .../test_redshift_connection.py | 38 ++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/tests_new/integration_tests/core/environment/global_conftest.py b/tests_new/integration_tests/core/environment/global_conftest.py index b8f32ceec..b9fc040ba 100644 --- a/tests_new/integration_tests/core/environment/global_conftest.py +++ b/tests_new/integration_tests/core/environment/global_conftest.py @@ -44,13 +44,14 @@ def delete_env(client, env): @pytest.fixture(scope='session') -def session_env1(client1, group1, org1, session_id, testdata): +def session_env1(client1, group1, group2, org1, session_id, testdata): envdata = testdata.envs['session_env1'] env = None try: env = create_env( client1, 'session_env1', group1, org1.organizationUri, envdata.accountId, envdata.region, tags=[session_id] ) + invite_group_on_env(client1, env.environmentUri, group2, ['CREATE_DATASET']) yield env finally: if env: diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py index 38597ca32..a45d7fa2f 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py +++ b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py @@ -155,49 +155,59 @@ def test_add_connection_group_permissions(client1, group2, session_connection_se assert_that(session_connection_serverless_admin_group_with_permissions).is_equal_to(group2) -def test_add_connection_group_permissions_unauthorized(client2, group3, session_connection_serverless_admin): +def test_add_connection_group_permissions_unauthorized(client2, group2, session_connection_serverless_admin): assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( client=client2, connection_uri=session_connection_serverless_admin.connectionUri, - group_uri=group3, + group_uri=group2, permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], ).contains( 'UnauthorizedOperation', - 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + 'EDIT_REDSHIFT_CONNECTION_PERMISSIONS', session_connection_serverless_admin.connectionUri, ) def test_add_connection_group_permissions_invalid_connection_type( - client1, group3, session_connection_serverless_data_user + client1, group2, session_connection_serverless_data_user ): assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( client=client1, connection_uri=session_connection_serverless_data_user.connectionUri, - group_uri=group3, + group_uri=group2, permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], ).contains('InvalidInput', 'ConnectionType', session_connection_serverless_data_user.connectionType) -def test_add_connection_group_permissions_invalid_permissions(client1, group3, session_connection_serverless_admin): +def test_add_connection_group_permissions_invalid_permissions(client1, group2, session_connection_serverless_admin): assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( client=client1, connection_uri=session_connection_serverless_admin.connectionUri, - group_uri=group3, + group_uri=group2, permissions=['INVALID_PERMISSION'], ).contains('InvalidInput', 'INVALID_PERMISSION', 'Permissions') -def test_delete_connection_group_permissions(client1, group3, session_connection_serverless_admin): - response = add_redshift_connection_group_permissions( +def test_add_connection_group_permissions_invalid_group(client1, group3, session_connection_serverless_admin): + assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group3, permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], + ).contains('InvalidInput', group3, 'Team') + + +def test_delete_connection_group_permissions(client1, group2, session_connection_serverless_admin): + response = delete_redshift_connection_group_permissions( + client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group2 ) assert_that(response).is_true() - response = delete_redshift_connection_group_permissions( - client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group3 + # Revert changes + response = add_redshift_connection_group_permissions( + client=client1, + connection_uri=session_connection_serverless_admin.connectionUri, + group_uri=group2, + permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], ) assert_that(response).is_true() @@ -209,7 +219,7 @@ def test_delete_connection_group_permissions_unauthorized(client2, group3, sessi group_uri=group3, ).contains( 'UnauthorizedOperation', - 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + 'EDIT_REDSHIFT_CONNECTION_PERMISSIONS', session_connection_serverless_admin.connectionUri, ) @@ -336,7 +346,7 @@ def test_list_redshift_connection_group_permissions_unauthorized(client2, sessio client=client2, connection_uri=session_connection_serverless_admin.connectionUri ).contains( 'UnauthorizedOperation', - 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + 'EDIT_REDSHIFT_CONNECTION_PERMISSIONS', session_connection_serverless_admin.connectionUri, ) @@ -357,6 +367,6 @@ def test_list_redshift_connection_group_no_permissions_unauthorized(client2, ses client=client2, connection_uri=session_connection_serverless_admin.connectionUri ).contains( 'UnauthorizedOperation', - 'MANAGE_REDSHIFT_CONNECTION_PERMISSIONS', + 'EDIT_REDSHIFT_CONNECTION_PERMISSIONS', session_connection_serverless_admin.connectionUri, ) From 0dd083f88122318ec00dcfbf7d6e9c7465de0e6b Mon Sep 17 00:00:00 2001 From: dlpzx Date: Tue, 15 Oct 2024 09:47:35 +0200 Subject: [PATCH 7/7] PR review comments --- .../core/environment/global_conftest.py | 4 ++-- .../redshift_datasets/global_conftest.py | 8 +++---- .../test_redshift_connection.py | 24 +++++++++---------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/tests_new/integration_tests/core/environment/global_conftest.py b/tests_new/integration_tests/core/environment/global_conftest.py index b9fc040ba..348c1b794 100644 --- a/tests_new/integration_tests/core/environment/global_conftest.py +++ b/tests_new/integration_tests/core/environment/global_conftest.py @@ -44,14 +44,14 @@ def delete_env(client, env): @pytest.fixture(scope='session') -def session_env1(client1, group1, group2, org1, session_id, testdata): +def session_env1(client1, group1, group5, org1, session_id, testdata): envdata = testdata.envs['session_env1'] env = None try: env = create_env( client1, 'session_env1', group1, org1.organizationUri, envdata.accountId, envdata.region, tags=[session_id] ) - invite_group_on_env(client1, env.environmentUri, group2, ['CREATE_DATASET']) + invite_group_on_env(client1, env.environmentUri, group5, ['CREATE_DATASET']) yield env finally: if env: diff --git a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py index ce58f3929..9532be262 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py +++ b/tests_new/integration_tests/modules/redshift_datasets/global_conftest.py @@ -68,20 +68,20 @@ def session_connection_serverless_admin(client1, group1, session_env1, testdata) @pytest.fixture(scope='session') -def session_connection_serverless_admin_group_with_permissions(client1, group2, session_connection_serverless_admin): +def session_connection_serverless_admin_group_with_permissions(client1, group5, session_connection_serverless_admin): permissions = None try: permissions = add_redshift_connection_group_permissions( client=client1, connection_uri=session_connection_serverless_admin.connectionUri, - group_uri=group2, + group_uri=group5, permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], ) - yield group2 + yield group5 finally: if permissions: delete_redshift_connection_group_permissions( - client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group2 + client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group5 ) diff --git a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py index a45d7fa2f..41b218487 100644 --- a/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py +++ b/tests_new/integration_tests/modules/redshift_datasets/test_redshift_connection.py @@ -151,15 +151,15 @@ def test_delete_connection_unauthorized(client2, session_connection_serverless_a ).contains('UnauthorizedOperation', 'DELETE_REDSHIFT_CONNECTION', session_connection_serverless_admin.connectionUri) -def test_add_connection_group_permissions(client1, group2, session_connection_serverless_admin_group_with_permissions): - assert_that(session_connection_serverless_admin_group_with_permissions).is_equal_to(group2) +def test_add_connection_group_permissions(client1, group5, session_connection_serverless_admin_group_with_permissions): + assert_that(session_connection_serverless_admin_group_with_permissions).is_equal_to(group5) -def test_add_connection_group_permissions_unauthorized(client2, group2, session_connection_serverless_admin): +def test_add_connection_group_permissions_unauthorized(client2, group5, session_connection_serverless_admin): assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( client=client2, connection_uri=session_connection_serverless_admin.connectionUri, - group_uri=group2, + group_uri=group5, permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], ).contains( 'UnauthorizedOperation', @@ -169,21 +169,21 @@ def test_add_connection_group_permissions_unauthorized(client2, group2, session_ def test_add_connection_group_permissions_invalid_connection_type( - client1, group2, session_connection_serverless_data_user + client1, group5, session_connection_serverless_data_user ): assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( client=client1, connection_uri=session_connection_serverless_data_user.connectionUri, - group_uri=group2, + group_uri=group5, permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], ).contains('InvalidInput', 'ConnectionType', session_connection_serverless_data_user.connectionType) -def test_add_connection_group_permissions_invalid_permissions(client1, group2, session_connection_serverless_admin): +def test_add_connection_group_permissions_invalid_permissions(client1, group5, session_connection_serverless_admin): assert_that(add_redshift_connection_group_permissions).raises(GqlError).when_called_with( client=client1, connection_uri=session_connection_serverless_admin.connectionUri, - group_uri=group2, + group_uri=group5, permissions=['INVALID_PERMISSION'], ).contains('InvalidInput', 'INVALID_PERMISSION', 'Permissions') @@ -197,16 +197,16 @@ def test_add_connection_group_permissions_invalid_group(client1, group3, session ).contains('InvalidInput', group3, 'Team') -def test_delete_connection_group_permissions(client1, group2, session_connection_serverless_admin): +def test_delete_connection_group_permissions(client1, group5, session_connection_serverless_admin): response = delete_redshift_connection_group_permissions( - client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group2 + client=client1, connection_uri=session_connection_serverless_admin.connectionUri, group_uri=group5 ) assert_that(response).is_true() # Revert changes response = add_redshift_connection_group_permissions( client=client1, connection_uri=session_connection_serverless_admin.connectionUri, - group_uri=group2, + group_uri=group5, permissions=['CREATE_SHARE_REQUEST_WITH_CONNECTION'], ) assert_that(response).is_true() @@ -354,7 +354,7 @@ def test_list_redshift_connection_group_permissions_unauthorized(client2, sessio def test_list_redshift_connection_group_no_permissions( client1, group1, session_connection_serverless_admin, session_connection_serverless_admin_group_with_permissions ): - response = list_redshift_connection_group_permissions( + response = list_redshift_connection_group_no_permissions( client=client1, connection_uri=session_connection_serverless_admin.connectionUri, )