Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(redshift): adding distKey to an existing table fails deployment #26789

Merged
merged 2 commits into from
Aug 17, 2023
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -164,9 +164,14 @@ async function updateTable(

const oldDistKey = getDistKeyColumn(oldTableColumns)?.name;
const newDistKey = getDistKeyColumn(tableColumns)?.name;
if ((!oldDistKey && newDistKey ) || (oldDistKey && !newDistKey)) {
return createTable(tableNamePrefix, tableNameSuffix, tableColumns, tableAndClusterProps);
if (!oldDistKey && newDistKey) {
// Table has no existing distribution key, add a new one
alterationStatements.push(`ALTER TABLE ${tableName} ALTER DISTSTYLE KEY DISTKEY ${newDistKey}`);
} else if (oldDistKey && !newDistKey) {
// Table has a distribution key, remove and set to AUTO
alterationStatements.push(`ALTER TABLE ${tableName} ALTER DISTSTYLE AUTO`);
} else if (oldDistKey !== newDistKey) {
// Table has an existing distribution key, change it
alterationStatements.push(`ALTER TABLE ${tableName} ALTER DISTKEY ${newDistKey}`);
}

Original file line number Diff line number Diff line change
@@ -384,21 +384,21 @@ describe('update', () => {
}));
});

test('replaces if distKey is added', async () => {
test('adds key without creating table if distKey is added', async () => {
const newResourceProperties: ResourcePropertiesType = {
...resourceProperties,
tableColumns: [{ name: 'col1', dataType: 'varchar(1)', distKey: true }],
};

await expect(manageTable(newResourceProperties, event)).resolves.not.toMatchObject({
await expect(manageTable(newResourceProperties, event)).resolves.toMatchObject({
PhysicalResourceId: physicalResourceId,
});
expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({
Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1)) DISTKEY(col1)`,
Sql: `ALTER TABLE ${physicalResourceId} ALTER DISTSTYLE KEY DISTKEY col1`,
}));
});

test('replaces if distKey is removed', async () => {
test('removes key without replacing table if distKey is removed', async () => {
const newEvent: AWSLambda.CloudFormationCustomResourceEvent = {
...event,
OldResourceProperties: {
@@ -410,11 +410,11 @@ describe('update', () => {
...resourceProperties,
};

await expect(manageTable(newResourceProperties, newEvent)).resolves.not.toMatchObject({
await expect(manageTable(newResourceProperties, newEvent)).resolves.toMatchObject({
PhysicalResourceId: physicalResourceId,
});
expect(mockExecuteStatement).toHaveBeenCalledWith(expect.objectContaining({
Sql: `CREATE TABLE ${tableNamePrefix}${requestIdTruncated} (col1 varchar(1))`,
Sql: `ALTER TABLE ${physicalResourceId} ALTER DISTSTYLE AUTO`,
}));
});

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"version": "33.0.0",
"files": {
"95dfca0d51dadef71390a414c093b29c52f39e1ff2acef2a26ded36e756bf386": {
"source": {
"path": "asset.95dfca0d51dadef71390a414c093b29c52f39e1ff2acef2a26ded36e756bf386",
"packaging": "zip"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "95dfca0d51dadef71390a414c093b29c52f39e1ff2acef2a26ded36e756bf386.zip",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
},
"73b60c2cf141bf58c33cfaa33858f5c84103a0232ba7192d696536488f7731c4": {
"source": {
"path": "asset.73b60c2cf141bf58c33cfaa33858f5c84103a0232ba7192d696536488f7731c4",
"packaging": "zip"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "73b60c2cf141bf58c33cfaa33858f5c84103a0232ba7192d696536488f7731c4.zip",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
},
"eb47b5477029fcb632fd7321d9e49e555755ddf5de0aff32fca72fdf28081319": {
"source": {
"path": "aws-cdk-redshift-distkey-create.template.json",
"packaging": "file"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "eb47b5477029fcb632fd7321d9e49e555755ddf5de0aff32fca72fdf28081319.json",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
}
},
"dockerImages": {}
}

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{
"version": "33.0.0",
"files": {
"95dfca0d51dadef71390a414c093b29c52f39e1ff2acef2a26ded36e756bf386": {
"source": {
"path": "asset.95dfca0d51dadef71390a414c093b29c52f39e1ff2acef2a26ded36e756bf386",
"packaging": "zip"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "95dfca0d51dadef71390a414c093b29c52f39e1ff2acef2a26ded36e756bf386.zip",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
},
"73b60c2cf141bf58c33cfaa33858f5c84103a0232ba7192d696536488f7731c4": {
"source": {
"path": "asset.73b60c2cf141bf58c33cfaa33858f5c84103a0232ba7192d696536488f7731c4",
"packaging": "zip"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "73b60c2cf141bf58c33cfaa33858f5c84103a0232ba7192d696536488f7731c4.zip",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
},
"65747181e631ebd4dcb64c9875500b198110a0ab7ccec1fce9d85799b12403af": {
"source": {
"path": "aws-cdk-redshift-distkey-update.template.json",
"packaging": "file"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "65747181e631ebd4dcb64c9875500b198110a0ab7ccec1fce9d85799b12403af.json",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
}
},
"dockerImages": {}
}

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"version": "33.0.0",
"files": {
"21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": {
"source": {
"path": "awscdkredshiftdistkeytestDefaultTestDeployAssert61A36BE8.template.json",
"packaging": "file"
},
"destinations": {
"current_account-current_region": {
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
"objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json",
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
}
}
}
},
"dockerImages": {}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
{
"Parameters": {
"BootstrapVersion": {
"Type": "AWS::SSM::Parameter::Value<String>",
"Default": "/cdk-bootstrap/hnb659fds/version",
"Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
}
},
"Rules": {
"CheckBootstrapVersion": {
"Assertions": [
{
"Assert": {
"Fn::Not": [
{
"Fn::Contains": [
[
"1",
"2",
"3",
"4",
"5"
],
{
"Ref": "BootstrapVersion"
}
]
}
]
},
"AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
}
]
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"version":"33.0.0"}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
{
"version": "33.0.0",
"testCases": {
"aws-cdk-redshift-distkey-test/DefaultTest": {
"stacks": [
"aws-cdk-redshift-distkey-create",
"aws-cdk-redshift-distkey-update"
],
"assertionStack": "aws-cdk-redshift-distkey-test/DefaultTest/DeployAssert",
"assertionStackName": "awscdkredshiftdistkeytestDefaultTestDeployAssert61A36BE8"
}
}
}

Large diffs are not rendered by default.

Large diffs are not rendered by default.

88 changes: 88 additions & 0 deletions packages/@aws-cdk/aws-redshift-alpha/test/integ.cluster-distkey.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import * as ec2 from 'aws-cdk-lib/aws-ec2';
import * as kms from 'aws-cdk-lib/aws-kms';
import * as cdk from 'aws-cdk-lib';
import * as integ from '@aws-cdk/integ-tests-alpha';
import * as constructs from 'constructs';
import * as redshift from '../lib';

/**
* This test does the following
*
* 1. Creates a stack with a Redshift cluster with a table without distkey
* 2. Creates a stack with the same cluster to update the table and cause the disktkey creation.
*/

const app = new cdk.App();

interface RedshiftDistKeyStackProps extends cdk.StackProps {
hasDistKey: boolean;
}

class RedshiftDistKeyStack extends cdk.Stack {
constructor(scope: constructs.Construct, id: string, props: RedshiftDistKeyStackProps) {
super(scope, id, props);

const key = new kms.Key(this, 'custom-kms-key');
const vpc = new ec2.Vpc(this, 'Vpc', { restrictDefaultSecurityGroup: false });
const databaseName = 'my_db';

const cluster = new redshift.Cluster(this, 'Cluster', {
vpc: vpc,
vpcSubnets: {
subnetType: ec2.SubnetType.PUBLIC,
},
masterUser: {
masterUsername: 'admin',
},
defaultDatabaseName: databaseName,
publiclyAccessible: true,
encryptionKey: key,
});

cluster.addToParameterGroup('enable_user_activity_logging', 'true');

const databaseOptions = {
cluster: cluster,
databaseName: databaseName,
};

const tableOptions = {
tableName: 'mytable',
sortStyle: redshift.TableSortStyle.AUTO,
tableComment: 'A test table',
};
new redshift.Table(this, 'Table', {
...databaseOptions,
tableColumns: [
{ name: 'col1', dataType: 'varchar(4)', distKey: props.hasDistKey, comment: 'A test column' },
],
...tableOptions,
});
}
}

const createStack = new RedshiftDistKeyStack(app, 'aws-cdk-redshift-distkey-create', {
hasDistKey: false,
});

const updateStack = new RedshiftDistKeyStack(app, 'aws-cdk-redshift-distkey-update', {
hasDistKey: true,
});

updateStack.addDependency(createStack);
const stacks = [createStack, updateStack];
stacks.forEach(s => {
cdk.Aspects.of(s).add({
visit(node: constructs.IConstruct) {
if (cdk.CfnResource.isCfnResource(node)) {
node.applyRemovalPolicy(cdk.RemovalPolicy.DESTROY);
}
},
});
});

new integ.IntegTest(app, 'aws-cdk-redshift-distkey-test', {
testCases: stacks,
});

app.synth();