From 97fe9d1e24c5d46f8dd3b233a87b4b9f058839cb Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 22 Apr 2020 23:57:28 +0300 Subject: [PATCH 1/6] fix(eks): version update completes prematurely The `UpdateClusterVersion` operation takes a while to begin and until then, the cluster's status is still `ACTIVE` instead `UPDATING` as expected. This causes the `isComplete` handler, which is called immediately, to think that the operation is complete, when it hasn't even began. Add logic to the cluster version update `onEvent` method to wait up to 5 minutes until the cluster status is no longer `ACTIVE`, so that the subsequent `isComplete` query will be based on the version update operation itself. Extended the timeout of `onEvent` to 15m to ensure it does not interrupt the operation. TESTING: Updated unit tests to verify this retry behavior and performed a manual upgrade tests while examining the logs. Fixes #7457 --- .../lib/cluster-resource-handler/cluster.ts | 35 +++++++++++++++++++ .../aws-eks/lib/cluster-resource-provider.ts | 2 +- packages/@aws-cdk/aws-eks/lib/cluster.ts | 2 +- .../test/test.cluster-resource-provider.ts | 16 +++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts index 9b7859b0c2e0e..475375b2ec1e6 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts @@ -159,6 +159,36 @@ export class ClusterResourceHandler extends ResourceHandler { } await this.eks.updateClusterVersion({ name: this.clusterName, version: newVersion }); + + // it take a while for the cluster to start the version update, and until + // then the cluster's status is still "ACTIVE", which causes version + // upgrades to complete prematurely. so, we wait here until the cluster + // status changes status from "ACTIVE" and only then yield execution to the + // async waiter. technically the status is expected to be "UPDATING", but it + // is more robust to just make sure it's not "ACTIVE" before we carry on + + // wait a total of 5 minutes for this to happen. + let remainingSec = 5 * 60; + + while (remainingSec > 0) { + console.log(`waiting for cluster to transition from ACTIVE status (remaining time: ${remainingSec}s)`); + const resp = await this.eks.describeCluster({ name: this.clusterName }); + console.log('describeCluster result:', JSON.stringify(resp, undefined, 2)); + + const status = resp.cluster?.status; + if (status !== 'ACTIVE') { + console.log(`cluster is now in ${status} state`); + break; + } + + // wait 2sec before trying again + await sleep(2000); + remainingSec -= 2; + } + + if (remainingSec === 0) { + throw new Error('version update failure: cluster did not transition from ACTIVE status after 5 minutes elapsed'); + } } private async isActive(): Promise { @@ -227,3 +257,8 @@ function analyzeUpdate(oldProps: Partial, newProps updateLogging: JSON.stringify(newProps.logging) !== JSON.stringify(oldProps.logging), }; } + +async function sleep(ms: number) { + console.log(`waiting ${ms} milliseconds`); + return new Promise(ok => setTimeout(ok, ms)); +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts index a3e3128f807b8..e7dfe30ed1522 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts @@ -41,7 +41,7 @@ export class ClusterResourceProvider extends NestedStack { description: 'onEvent handler for EKS cluster resource provider', runtime: HANDLER_RUNTIME, handler: 'index.onEvent', - timeout: Duration.minutes(1), + timeout: Duration.minutes(15), }); const isComplete = new lambda.Function(this, 'IsCompleteHandler', { diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 11013d34fc9a0..d035e341f1fde 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -1078,6 +1078,6 @@ export enum MachineImageType { const GPU_INSTANCETYPES = ['p2', 'p3', 'g4']; -export function nodeTypeForInstanceType(instanceType: ec2.InstanceType) { +function nodeTypeForInstanceType(instanceType: ec2.InstanceType) { return GPU_INSTANCETYPES.includes(instanceType.toString().substring(0, 2)) ? NodeType.GPU : NodeType.STANDARD; } diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts b/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts index 33a7431894572..5d357373b00dc 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts @@ -361,13 +361,21 @@ export = { }, { version: undefined, })); + + let count = 3; + const restore = mocks.client.describeCluster; + mocks.client.describeCluster = async () => ({ + cluster: { status: --count === 0 ? 'UPDATING' : 'ACTIVE' }, + }); const resp = await handler.onEvent(); + mocks.client.describeCluster = restore; test.equal(resp, undefined); test.deepEqual(mocks.actualRequest.updateClusterVersionRequest!, { name: 'physical-resource-id', version: '12.34', }); test.equal(mocks.actualRequest.createClusterRequest, undefined); + test.equal(count, 0); // make sure "describeCluster" was called until it returned 'UPDATING' test.done(); }, @@ -377,13 +385,21 @@ export = { }, { version: '1.1', })); + + let count = 3; + const restore = mocks.client.describeCluster; + mocks.client.describeCluster = async () => ({ + cluster: { status: --count === 0 ? 'UPDATING' : 'ACTIVE' }, + }); const resp = await handler.onEvent(); + mocks.client.describeCluster = restore; test.equal(resp, undefined); test.deepEqual(mocks.actualRequest.updateClusterVersionRequest!, { name: 'physical-resource-id', version: '2.0', }); test.equal(mocks.actualRequest.createClusterRequest, undefined); + test.equal(count, 0); // make sure "describeCluster" was called until it returned 'UPDATING' test.done(); }, From 258fbe7d5c67fdef773ae8b25a67e23b19689548 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 23 Apr 2020 09:29:29 +0300 Subject: [PATCH 2/6] update expectation --- .../test/integ.eks-cluster.expected.json | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index 1f88a2c6b34b0..e4348d7c4931a 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -2231,7 +2231,7 @@ }, "/", { - "Ref": "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3Bucket835D19A2" + "Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3BucketA7931423" }, "/", { @@ -2241,7 +2241,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3VersionKeyBFF2DA61" + "Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC" } ] } @@ -2254,7 +2254,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3VersionKeyBFF2DA61" + "Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC" } ] } @@ -2264,17 +2264,17 @@ ] }, "Parameters": { - "referencetoawscdkeksclustertestAssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3Bucket57C4C68FRef": { - "Ref": "AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3Bucket7CCCFC30" + "referencetoawscdkeksclustertestAssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket7AF1B74FRef": { + "Ref": "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket934A7DFE" }, - "referencetoawscdkeksclustertestAssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3VersionKeyBB973CE7Ref": { - "Ref": "AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3VersionKeyA2A28538" + "referencetoawscdkeksclustertestAssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey2AABFF64Ref": { + "Ref": "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey51C84156" }, - "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3BucketC7CBF350Ref": { - "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C" + "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3Bucket9CFB351FRef": { + "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F" }, - "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKey7E2BE411Ref": { - "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB" + "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKey8BE6EE20Ref": { + "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF" } } } @@ -2292,7 +2292,7 @@ }, "/", { - "Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3Bucket2D824DEF" + "Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3BucketA33B0C7A" }, "/", { @@ -2302,7 +2302,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4" + "Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E" } ] } @@ -2315,7 +2315,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4" + "Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E" } ] } @@ -2331,11 +2331,11 @@ "referencetoawscdkeksclustertestAssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3VersionKeyA18C5C39Ref": { "Ref": "AssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3VersionKeyBED95764" }, - "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3BucketC7CBF350Ref": { - "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C" + "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3Bucket9CFB351FRef": { + "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F" }, - "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKey7E2BE411Ref": { - "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB" + "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKey8BE6EE20Ref": { + "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF" } } } @@ -2413,29 +2413,29 @@ } }, "Parameters": { - "AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3Bucket7CCCFC30": { + "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket934A7DFE": { "Type": "String", - "Description": "S3 bucket for asset \"54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8\"" + "Description": "S3 bucket for asset \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\"" }, - "AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8S3VersionKeyA2A28538": { + "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey51C84156": { "Type": "String", - "Description": "S3 key for asset version \"54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8\"" + "Description": "S3 key for asset version \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\"" }, - "AssetParameters54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8ArtifactHashC0D1FA2A": { + "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730ArtifactHashE97DF9EF": { "Type": "String", - "Description": "Artifact hash for asset \"54c9eae68c19c65a224969094e8447eed31d811384c7e32bdb72fffb4be15ac8\"" + "Description": "Artifact hash for asset \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\"" }, - "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C": { + "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F": { "Type": "String", - "Description": "S3 bucket for asset \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\"" + "Description": "S3 bucket for asset \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\"" }, - "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB": { + "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF": { "Type": "String", - "Description": "S3 key for asset version \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\"" + "Description": "S3 key for asset version \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\"" }, - "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1ArtifactHash251241BC": { + "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984ArtifactHash67844294": { "Type": "String", - "Description": "Artifact hash for asset \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\"" + "Description": "Artifact hash for asset \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\"" }, "AssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3Bucket0C3A00C2": { "Type": "String", @@ -2449,29 +2449,29 @@ "Type": "String", "Description": "Artifact hash for asset \"a6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afb\"" }, - "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3Bucket835D19A2": { + "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3BucketA7931423": { "Type": "String", - "Description": "S3 bucket for asset \"5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69\"" + "Description": "S3 bucket for asset \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\"" }, - "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69S3VersionKeyBFF2DA61": { + "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC": { "Type": "String", - "Description": "S3 key for asset version \"5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69\"" + "Description": "S3 key for asset version \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\"" }, - "AssetParameters5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69ArtifactHash0E7708FD": { + "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7ArtifactHash7CFF23C4": { "Type": "String", - "Description": "Artifact hash for asset \"5c7de45abd07f88cf62deefa6399553786f3559084ff34bae66042cdd1987d69\"" + "Description": "Artifact hash for asset \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\"" }, - "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3Bucket2D824DEF": { + "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3BucketA33B0C7A": { "Type": "String", - "Description": "S3 bucket for asset \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\"" + "Description": "S3 bucket for asset \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\"" }, - "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4": { + "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E": { "Type": "String", - "Description": "S3 key for asset version \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\"" + "Description": "S3 key for asset version \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\"" }, - "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcArtifactHash83AE269A": { + "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bArtifactHashBB584529": { "Type": "String", - "Description": "Artifact hash for asset \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\"" + "Description": "Artifact hash for asset \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\"" }, "SsmParameterValueawsserviceeksoptimizedami114amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": { "Type": "AWS::SSM::Parameter::Value", From aeeb3841d094f38534e3f9837c51482df6b32583 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 23 Apr 2020 15:09:06 +0300 Subject: [PATCH 3/6] custom-resources: formalize the notion of passing arbitrary fields to isComplete This was already supported, just add some docs and tests to make sure this continues to be supported. --- packages/@aws-cdk/custom-resources/README.md | 9 +++---- .../lib/provider-framework/types.d.ts | 5 ++++ .../integ.provider.expected.json | 24 +++++++++---------- .../s3-assert-handler/index.py | 7 ++++++ .../test/provider-framework/runtime.test.ts | 2 ++ 5 files changed, 31 insertions(+), 16 deletions(-) diff --git a/packages/@aws-cdk/custom-resources/README.md b/packages/@aws-cdk/custom-resources/README.md index d30028903575d..7d73424d9d12e 100644 --- a/packages/@aws-cdk/custom-resources/README.md +++ b/packages/@aws-cdk/custom-resources/README.md @@ -137,6 +137,7 @@ The return value from `onEvent` must be a JSON object with the following fields: |-----|----|--------|----------- |`PhysicalResourceId`|String|No|The allocated/assigned physical ID of the resource. If omitted for `Create` events, the event's `RequestId` will be used. For `Update`, the current physical ID will be used. If a different value is returned, CloudFormation will follow with a subsequent `Delete` for the previous ID (resource replacement). For `Delete`, it will always return the current physical resource ID, and if the user returns a different one, an error will occur. |`Data`|JSON|No|Resource attributes, which can later be retrieved through `Fn::GetAtt` on the custom resource object. +|*any*|*any*|No|Any other field included in the response will be passed through to `isComplete`. This can sometimes be useful to pass state between the handlers. [Custom Resource Provider Request]: https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/crpg-ref-requests.html#crpg-ref-request-fields @@ -158,10 +159,10 @@ with the message "Operation timed out". If an error is thrown, the framework will submit a "FAILED" response to AWS CloudFormation. -The input event to `isComplete` is similar to -[`onEvent`](#handling-lifecycle-events-onevent), with an additional guarantee -that `PhysicalResourceId` is defines and contains the value returned from -`onEvent` or the described default. At any case, it is guaranteed to exist. +The input event to `isComplete` includes all request fields, combined with all +fields returned from `onEvent`. If `PhysicalResourceId` has not been explicitly +returned from `onEvent`, it's value will be calculated based on the heuristics +described above. The return value must be a JSON object with the following fields: diff --git a/packages/@aws-cdk/custom-resources/lib/provider-framework/types.d.ts b/packages/@aws-cdk/custom-resources/lib/provider-framework/types.d.ts index 058405bf4e928..33a125a971cca 100644 --- a/packages/@aws-cdk/custom-resources/lib/provider-framework/types.d.ts +++ b/packages/@aws-cdk/custom-resources/lib/provider-framework/types.d.ts @@ -75,6 +75,11 @@ interface OnEventResponse { * Resource attributes to return. */ readonly Data?: { [name: string]: any }; + + /** + * Custom fields returned from OnEvent will be passed to IsComplete. + */ + readonly [key: string]: any; } /** diff --git a/packages/@aws-cdk/custom-resources/test/provider-framework/integ.provider.expected.json b/packages/@aws-cdk/custom-resources/test/provider-framework/integ.provider.expected.json index a2307af6ac1ab..9907ab690dd70 100644 --- a/packages/@aws-cdk/custom-resources/test/provider-framework/integ.provider.expected.json +++ b/packages/@aws-cdk/custom-resources/test/provider-framework/integ.provider.expected.json @@ -340,7 +340,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3Bucket776FD46E" + "Ref": "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3Bucket0DB889DF" }, "S3Key": { "Fn::Join": [ @@ -353,7 +353,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3VersionKeyA70347F9" + "Ref": "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3VersionKey67FE4034" } ] } @@ -366,7 +366,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3VersionKeyA70347F9" + "Ref": "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3VersionKey67FE4034" } ] } @@ -450,7 +450,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3Bucket776FD46E" + "Ref": "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3Bucket0DB889DF" }, "S3Key": { "Fn::Join": [ @@ -463,7 +463,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3VersionKeyA70347F9" + "Ref": "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3VersionKey67FE4034" } ] } @@ -476,7 +476,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3VersionKeyA70347F9" + "Ref": "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3VersionKey67FE4034" } ] } @@ -1054,17 +1054,17 @@ "Type": "String", "Description": "Artifact hash for asset \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\"" }, - "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3Bucket776FD46E": { + "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3Bucket0DB889DF": { "Type": "String", - "Description": "S3 bucket for asset \"db961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511\"" + "Description": "S3 bucket for asset \"4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8\"" }, - "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511S3VersionKeyA70347F9": { + "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8S3VersionKey67FE4034": { "Type": "String", - "Description": "S3 key for asset version \"db961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511\"" + "Description": "S3 key for asset version \"4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8\"" }, - "AssetParametersdb961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511ArtifactHashB3EA6E4A": { + "AssetParameters4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8ArtifactHash6C17CFC2": { "Type": "String", - "Description": "Artifact hash for asset \"db961fc9d087616ad76339bd5135f518cea24001f866a17067a1024235128511\"" + "Description": "Artifact hash for asset \"4bafad8d010ba693e235b77d2c6decfc2ac79a8208d4477cbb36d31caf7189e8\"" } }, "Outputs": { diff --git a/packages/@aws-cdk/custom-resources/test/provider-framework/integration-test-fixtures/s3-assert-handler/index.py b/packages/@aws-cdk/custom-resources/test/provider-framework/integration-test-fixtures/s3-assert-handler/index.py index a5e9321c895ea..0f99bdca49aa9 100644 --- a/packages/@aws-cdk/custom-resources/test/provider-framework/integration-test-fixtures/s3-assert-handler/index.py +++ b/packages/@aws-cdk/custom-resources/test/provider-framework/integration-test-fixtures/s3-assert-handler/index.py @@ -4,10 +4,17 @@ def on_event(event, ctx): print(event) + return { + 'ArbitraryField': 12345 + } def is_complete(event, ctx): print(event) + # verify result from on_event is passed through + if event.get('ArbitraryField', None) != 12345: + raise 'Error: expecting "event" to include "ArbitraryField" with value 12345' + # nothing to assert if this resource is being deleted if event['RequestType'] == 'Delete': return { 'IsComplete': True } diff --git a/packages/@aws-cdk/custom-resources/test/provider-framework/runtime.test.ts b/packages/@aws-cdk/custom-resources/test/provider-framework/runtime.test.ts index 0f1679955ed06..3898246fab6c4 100644 --- a/packages/@aws-cdk/custom-resources/test/provider-framework/runtime.test.ts +++ b/packages/@aws-cdk/custom-resources/test/provider-framework/runtime.test.ts @@ -32,11 +32,13 @@ test('async flow: isComplete returns true only after 3 times', async () => { return { PhysicalResourceId: MOCK_PHYSICAL_ID, Data: MOCK_ATTRS, + ArbitraryField: 1234, }; }; mocks.isCompleteImplMock = async event => { isCompleteCalls++; + expect((event as any).ArbitraryField).toEqual(1234); // any field is passed through expect(event.PhysicalResourceId).toEqual(MOCK_PHYSICAL_ID); // physical ID returned from onEvent is passed to "isComplete" expect(event.Data).toStrictEqual(MOCK_ATTRS); // attributes are propagated between the calls From 4f18dc3ff0cde2a348fe0e29acb9165fd9eb7964 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 23 Apr 2020 15:09:18 +0300 Subject: [PATCH 4/6] use describeUpdate instead of cluster status --- .../lib/cluster-resource-handler/cluster.ts | 81 ++++++++--------- .../lib/cluster-resource-handler/common.ts | 18 +++- .../lib/cluster-resource-handler/index.ts | 1 + .../test/cluster-resource-handler-mocks.ts | 29 ++++++- .../test/integ.eks-cluster.expected.json | 84 +++++++++--------- .../test/test.cluster-resource-provider.ts | 86 +++++++++++++++---- .../test/test.fargate-resource-provider.ts | 1 + 7 files changed, 196 insertions(+), 104 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts index 475375b2ec1e6..3d0c8c25e39a3 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/cluster.ts @@ -3,7 +3,7 @@ import { IsCompleteResponse, OnEventResponse } from '@aws-cdk/custom-resources/lib/provider-framework/types'; // eslint-disable-next-line import/no-extraneous-dependencies import * as aws from 'aws-sdk'; -import { EksClient, ResourceHandler } from './common'; +import { EksClient, ResourceEvent, ResourceHandler } from './common'; const MAX_CLUSTER_NAME_LEN = 100; @@ -19,7 +19,7 @@ export class ClusterResourceHandler extends ResourceHandler { private readonly newProps: aws.EKS.CreateClusterRequest; private readonly oldProps: Partial; - constructor(eks: EksClient, event: AWSLambda.CloudFormationCustomResourceEvent) { + constructor(eks: EksClient, event: ResourceEvent) { super(eks, event); this.newProps = parseProps(this.event.ResourceProperties); @@ -127,15 +127,17 @@ export class ClusterResourceHandler extends ResourceHandler { throw new Error(`Cannot remove cluster version configuration. Current version is ${this.oldProps.version}`); } - await this.updateClusterVersion(this.newProps.version); + return await this.updateClusterVersion(this.newProps.version); } if (updates.updateLogging || updates.updateAccess) { - await this.eks.updateClusterConfig({ + const updateResponse = await this.eks.updateClusterConfig({ name: this.clusterName, logging: this.newProps.logging, resourcesVpcConfig: this.newProps.resourcesVpcConfig, }); + + return { EksUpdateId: updateResponse.update?.id }; } // no updates @@ -144,6 +146,12 @@ export class ClusterResourceHandler extends ResourceHandler { protected async isUpdateComplete() { console.log('isUpdateComplete'); + + // if this is an EKS update, we will monitor the update event itself + if (this.event.EksUpdateId) { + return this.isEksUpdateComplete(this.event.EksUpdateId); + } + return this.isActive(); } @@ -158,37 +166,8 @@ export class ClusterResourceHandler extends ResourceHandler { return; } - await this.eks.updateClusterVersion({ name: this.clusterName, version: newVersion }); - - // it take a while for the cluster to start the version update, and until - // then the cluster's status is still "ACTIVE", which causes version - // upgrades to complete prematurely. so, we wait here until the cluster - // status changes status from "ACTIVE" and only then yield execution to the - // async waiter. technically the status is expected to be "UPDATING", but it - // is more robust to just make sure it's not "ACTIVE" before we carry on - - // wait a total of 5 minutes for this to happen. - let remainingSec = 5 * 60; - - while (remainingSec > 0) { - console.log(`waiting for cluster to transition from ACTIVE status (remaining time: ${remainingSec}s)`); - const resp = await this.eks.describeCluster({ name: this.clusterName }); - console.log('describeCluster result:', JSON.stringify(resp, undefined, 2)); - - const status = resp.cluster?.status; - if (status !== 'ACTIVE') { - console.log(`cluster is now in ${status} state`); - break; - } - - // wait 2sec before trying again - await sleep(2000); - remainingSec -= 2; - } - - if (remainingSec === 0) { - throw new Error('version update failure: cluster did not transition from ACTIVE status after 5 minutes elapsed'); - } + const updateResponse = await this.eks.updateClusterVersion({ name: this.clusterName, version: newVersion }); + return { EksUpdateId: updateResponse.update?.id }; } private async isActive(): Promise { @@ -217,6 +196,33 @@ export class ClusterResourceHandler extends ResourceHandler { } } + private async isEksUpdateComplete(eksUpdateId: string) { + this.log({ isEksUpdateComplete: eksUpdateId }); + + const describeUpdateResponse = await this.eks.describeUpdate({ + name: this.clusterName, + updateId: eksUpdateId, + }); + + this.log({ describeUpdateResponse }); + + if (!describeUpdateResponse.update) { + throw new Error(`unable to describe update with id "${eksUpdateId}"`); + } + + switch (describeUpdateResponse.update.status) { + case 'InProgress': + return { IsComplete: false }; + case 'Successful': + return { IsComplete: true }; + case 'Failed': + case 'Cancelled': + throw new Error(`cluster update id "${eksUpdateId}" failed with errors: ${JSON.stringify(describeUpdateResponse.update.errors)}`); + default: + throw new Error(`unknown status "${describeUpdateResponse.update.status}" for update id "${eksUpdateId}"`); + } + } + private generateClusterName() { const suffix = this.requestId.replace(/-/g, ''); // 32 chars const prefix = this.logicalResourceId.substr(0, MAX_CLUSTER_NAME_LEN - suffix.length - 1); @@ -257,8 +263,3 @@ function analyzeUpdate(oldProps: Partial, newProps updateLogging: JSON.stringify(newProps.logging) !== JSON.stringify(oldProps.logging), }; } - -async function sleep(ms: number) { - console.log(`waiting ${ms} milliseconds`); - return new Promise(ok => setTimeout(ok, ms)); -} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts index 1349563bf0996..57d3ae20f8cef 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/common.ts @@ -3,14 +3,25 @@ import { IsCompleteResponse, OnEventResponse } from '@aws-cdk/custom-resources/l // eslint-disable-next-line import/no-extraneous-dependencies import * as aws from 'aws-sdk'; +export interface EksUpdateId { + /** + * If this field is included in an event passed to "IsComplete", it means we + * initiated an EKS update that should be monitored using eks:DescribeUpdate + * instead of just looking at the cluster status. + */ + EksUpdateId?: string +} + +export type ResourceEvent = AWSLambda.CloudFormationCustomResourceEvent & EksUpdateId; + export abstract class ResourceHandler { protected readonly requestId: string; protected readonly logicalResourceId: string; protected readonly requestType: 'Create' | 'Update' | 'Delete'; protected readonly physicalResourceId?: string; - protected readonly event: AWSLambda.CloudFormationCustomResourceEvent; + protected readonly event: ResourceEvent; - constructor(protected readonly eks: EksClient, event: AWSLambda.CloudFormationCustomResourceEvent) { + constructor(protected readonly eks: EksClient, event: ResourceEvent) { this.requestType = event.RequestType; this.requestId = event.RequestId; this.logicalResourceId = event.LogicalResourceId; @@ -55,7 +66,7 @@ export abstract class ResourceHandler { protected abstract async onCreate(): Promise; protected abstract async onDelete(): Promise; - protected abstract async onUpdate(): Promise; + protected abstract async onUpdate(): Promise<(OnEventResponse & EksUpdateId) | void>; protected abstract async isCreateComplete(): Promise; protected abstract async isDeleteComplete(): Promise; protected abstract async isUpdateComplete(): Promise; @@ -68,6 +79,7 @@ export interface EksClient { describeCluster(request: aws.EKS.DescribeClusterRequest): Promise; updateClusterConfig(request: aws.EKS.UpdateClusterConfigRequest): Promise; updateClusterVersion(request: aws.EKS.UpdateClusterVersionRequest): Promise; + describeUpdate(req: aws.EKS.DescribeUpdateRequest): Promise; createFargateProfile(request: aws.EKS.CreateFargateProfileRequest): Promise; describeFargateProfile(request: aws.EKS.DescribeFargateProfileRequest): Promise; deleteFargateProfile(request: aws.EKS.DeleteFargateProfileRequest): Promise; diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts index 7e12bc72b411f..f2b796297246a 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-handler/index.ts @@ -16,6 +16,7 @@ const defaultEksClient: EksClient = { createCluster: req => getEksClient().createCluster(req).promise(), deleteCluster: req => getEksClient().deleteCluster(req).promise(), describeCluster: req => getEksClient().describeCluster(req).promise(), + describeUpdate: req => getEksClient().describeUpdate(req).promise(), updateClusterConfig: req => getEksClient().updateClusterConfig(req).promise(), updateClusterVersion: req => getEksClient().updateClusterVersion(req).promise(), createFargateProfile: req => getEksClient().createFargateProfile(req).promise(), diff --git a/packages/@aws-cdk/aws-eks/test/cluster-resource-handler-mocks.ts b/packages/@aws-cdk/aws-eks/test/cluster-resource-handler-mocks.ts index 7bcc866024418..c7980e0a89cf4 100644 --- a/packages/@aws-cdk/aws-eks/test/cluster-resource-handler-mocks.ts +++ b/packages/@aws-cdk/aws-eks/test/cluster-resource-handler-mocks.ts @@ -9,6 +9,7 @@ export let actualRequest: { configureAssumeRoleRequest?: sdk.STS.AssumeRoleRequest; createClusterRequest?: sdk.EKS.CreateClusterRequest; describeClusterRequest?: sdk.EKS.DescribeClusterRequest; + describeUpdateRequest?: sdk.EKS.DescribeUpdateRequest; deleteClusterRequest?: sdk.EKS.DeleteClusterRequest; updateClusterConfigRequest?: sdk.EKS.UpdateClusterConfigRequest; updateClusterVersionRequest?: sdk.EKS.UpdateClusterVersionRequest; @@ -22,6 +23,8 @@ export let actualRequest: { */ export let simulateResponse: { describeClusterResponseMockStatus?: string; + describeUpdateResponseMockStatus?: string; + describeUpdateResponseMockErrors?: sdk.EKS.ErrorDetails; deleteClusterErrorCode?: string; describeClusterExceptionCode?: string; } = { }; @@ -31,6 +34,8 @@ export function reset() { simulateResponse = { }; } +export const MOCK_UPDATE_STATUS_ID = 'MockEksUpdateStatusId'; + export const client: EksClient = { configureAssumeRole: req => { @@ -87,14 +92,34 @@ export const client: EksClient = { }; }, + describeUpdate: async req => { + actualRequest.describeUpdateRequest = req; + + return { + update: { + id: req.updateId, + errors: simulateResponse.describeUpdateResponseMockErrors, + status: simulateResponse.describeUpdateResponseMockStatus, + }, + }; + }, + updateClusterConfig: async req => { actualRequest.updateClusterConfigRequest = req; - return { }; + return { + update: { + id: MOCK_UPDATE_STATUS_ID, + }, + }; }, updateClusterVersion: async req => { actualRequest.updateClusterVersionRequest = req; - return { }; + return { + update: { + id: MOCK_UPDATE_STATUS_ID, + }, + }; }, createFargateProfile: async req => { diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index e4348d7c4931a..6d59deb5a14a9 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -2231,7 +2231,7 @@ }, "/", { - "Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3BucketA7931423" + "Ref": "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3Bucket889DD28D" }, "/", { @@ -2241,7 +2241,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC" + "Ref": "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3VersionKey9765793B" } ] } @@ -2254,7 +2254,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC" + "Ref": "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3VersionKey9765793B" } ] } @@ -2264,17 +2264,17 @@ ] }, "Parameters": { - "referencetoawscdkeksclustertestAssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket7AF1B74FRef": { - "Ref": "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket934A7DFE" + "referencetoawscdkeksclustertestAssetParametersc0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602S3Bucket38D74D5ERef": { + "Ref": "AssetParametersc0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602S3BucketD006BE3B" }, - "referencetoawscdkeksclustertestAssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey2AABFF64Ref": { - "Ref": "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey51C84156" + "referencetoawscdkeksclustertestAssetParametersc0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602S3VersionKey189EBCBARef": { + "Ref": "AssetParametersc0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602S3VersionKeyEC71339F" }, - "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3Bucket9CFB351FRef": { - "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F" + "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3BucketC7CBF350Ref": { + "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C" }, - "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKey8BE6EE20Ref": { - "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF" + "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKey7E2BE411Ref": { + "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB" } } } @@ -2292,7 +2292,7 @@ }, "/", { - "Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3BucketA33B0C7A" + "Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3Bucket2D824DEF" }, "/", { @@ -2302,7 +2302,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E" + "Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4" } ] } @@ -2315,7 +2315,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E" + "Ref": "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4" } ] } @@ -2331,11 +2331,11 @@ "referencetoawscdkeksclustertestAssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3VersionKeyA18C5C39Ref": { "Ref": "AssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3VersionKeyBED95764" }, - "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3Bucket9CFB351FRef": { - "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F" + "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3BucketC7CBF350Ref": { + "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C" }, - "referencetoawscdkeksclustertestAssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKey8BE6EE20Ref": { - "Ref": "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF" + "referencetoawscdkeksclustertestAssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKey7E2BE411Ref": { + "Ref": "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB" } } } @@ -2413,29 +2413,29 @@ } }, "Parameters": { - "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3Bucket934A7DFE": { + "AssetParametersc0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602S3BucketD006BE3B": { "Type": "String", - "Description": "S3 bucket for asset \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\"" + "Description": "S3 bucket for asset \"c0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602\"" }, - "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730S3VersionKey51C84156": { + "AssetParametersc0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602S3VersionKeyEC71339F": { "Type": "String", - "Description": "S3 key for asset version \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\"" + "Description": "S3 key for asset version \"c0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602\"" }, - "AssetParameters1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730ArtifactHashE97DF9EF": { + "AssetParametersc0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602ArtifactHash35F5D0CC": { "Type": "String", - "Description": "Artifact hash for asset \"1a900ca2314e1f584fd13febe7acbd10481693b30bbdfb9fe3ccbde1875f2730\"" + "Description": "Artifact hash for asset \"c0e453b77d5ccf090915fba7c771380f8370da5cbcc3c7ed757c98addd75b602\"" }, - "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3BucketD6AB456F": { + "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3Bucket663A709C": { "Type": "String", - "Description": "S3 bucket for asset \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\"" + "Description": "S3 bucket for asset \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\"" }, - "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984S3VersionKeyBE53D6BF": { + "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1S3VersionKeyF33697EB": { "Type": "String", - "Description": "S3 key for asset version \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\"" + "Description": "S3 key for asset version \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\"" }, - "AssetParameters4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984ArtifactHash67844294": { + "AssetParameters5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1ArtifactHash251241BC": { "Type": "String", - "Description": "Artifact hash for asset \"4643232167f9dcde888e1351b759b352d3351367a382d94a45db4ec89ba79984\"" + "Description": "Artifact hash for asset \"5e49cf64d8027f48872790f80cdb76c5b836ecf9a70b71be1eb937a5c25a47c1\"" }, "AssetParametersa6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afbS3Bucket0C3A00C2": { "Type": "String", @@ -2449,29 +2449,29 @@ "Type": "String", "Description": "Artifact hash for asset \"a6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afb\"" }, - "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3BucketA7931423": { + "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3Bucket889DD28D": { "Type": "String", - "Description": "S3 bucket for asset \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\"" + "Description": "S3 bucket for asset \"41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16e\"" }, - "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7S3VersionKey0FDEA8BC": { + "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3VersionKey9765793B": { "Type": "String", - "Description": "S3 key for asset version \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\"" + "Description": "S3 key for asset version \"41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16e\"" }, - "AssetParameters31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7ArtifactHash7CFF23C4": { + "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eArtifactHash1A2B5996": { "Type": "String", - "Description": "Artifact hash for asset \"31f982bcfc9c9d4e528c8e02ac3fd71ed01d0bd2c2b93078389208fa8c0bdeb7\"" + "Description": "Artifact hash for asset \"41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16e\"" }, - "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3BucketA33B0C7A": { + "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3Bucket2D824DEF": { "Type": "String", - "Description": "S3 bucket for asset \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\"" + "Description": "S3 bucket for asset \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\"" }, - "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bS3VersionKey07460D5E": { + "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3VersionKey45D8E8E4": { "Type": "String", - "Description": "S3 key for asset version \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\"" + "Description": "S3 key for asset version \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\"" }, - "AssetParameters212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1bArtifactHashBB584529": { + "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcArtifactHash83AE269A": { "Type": "String", - "Description": "Artifact hash for asset \"212ddf84fc36a377ecc1eedc6cb8355122619dc656e0a5847b6eb12c42403b1b\"" + "Description": "Artifact hash for asset \"36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbc\"" }, "SsmParameterValueawsserviceeksoptimizedami114amazonlinux2recommendedimageidC96584B6F00A464EAD1953AFF4B05118Parameter": { "Type": "AWS::SSM::Parameter::Value", diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts b/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts index 5d357373b00dc..0759704acead4 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster-resource-provider.ts @@ -353,7 +353,74 @@ export = { }, }, + 'isUpdateComplete with EKS update ID': { + + async 'with "Failed" status'(test: Test) { + const event = mocks.newRequest('Update'); + const isCompleteHandler = new ClusterResourceHandler(mocks.client, { + ...event, + EksUpdateId: 'foobar', + }); + + mocks.simulateResponse.describeUpdateResponseMockStatus = 'Failed'; + mocks.simulateResponse.describeUpdateResponseMockErrors = [ + { + errorMessage: 'errorMessageMock', + errorCode: 'errorCodeMock', + resourceIds: [ + 'foo', 'bar', + ], + }, + ]; + + let error; + try { + await isCompleteHandler.isComplete(); + } catch (e) { + error = e; + } + test.ok(error); + test.deepEqual(mocks.actualRequest.describeUpdateRequest, { name: 'physical-resource-id', updateId: 'foobar' }); + test.equal(error.message, 'cluster update id "foobar" failed with errors: [{"errorMessage":"errorMessageMock","errorCode":"errorCodeMock","resourceIds":["foo","bar"]}]'); + test.done(); + }, + + async 'with "InProgress" status, returns IsComplete=false'(test: Test) { + const event = mocks.newRequest('Update'); + const isCompleteHandler = new ClusterResourceHandler(mocks.client, { + ...event, + EksUpdateId: 'foobar', + }); + + mocks.simulateResponse.describeUpdateResponseMockStatus = 'InProgress'; + + const response = await isCompleteHandler.isComplete(); + + test.deepEqual(mocks.actualRequest.describeUpdateRequest, { name: 'physical-resource-id', updateId: 'foobar' }); + test.equal(response.IsComplete, false); + test.done(); + }, + + async 'with "Successful" status, returns IsComplete=true'(test: Test) { + const event = mocks.newRequest('Update'); + const isCompleteHandler = new ClusterResourceHandler(mocks.client, { + ...event, + EksUpdateId: 'foobar', + }); + + mocks.simulateResponse.describeUpdateResponseMockStatus = 'Successful'; + + const response = await isCompleteHandler.isComplete(); + + test.deepEqual(mocks.actualRequest.describeUpdateRequest, { name: 'physical-resource-id', updateId: 'foobar' }); + test.equal(response.IsComplete, true); + test.done(); + }, + + }, + 'in-place': { + 'version change': { async 'from undefined to a specific value'(test: Test) { const handler = new ClusterResourceHandler(mocks.client, mocks.newRequest('Update', { @@ -361,21 +428,13 @@ export = { }, { version: undefined, })); - - let count = 3; - const restore = mocks.client.describeCluster; - mocks.client.describeCluster = async () => ({ - cluster: { status: --count === 0 ? 'UPDATING' : 'ACTIVE' }, - }); const resp = await handler.onEvent(); - mocks.client.describeCluster = restore; - test.equal(resp, undefined); + test.deepEqual(resp, { EksUpdateId: mocks.MOCK_UPDATE_STATUS_ID }); test.deepEqual(mocks.actualRequest.updateClusterVersionRequest!, { name: 'physical-resource-id', version: '12.34', }); test.equal(mocks.actualRequest.createClusterRequest, undefined); - test.equal(count, 0); // make sure "describeCluster" was called until it returned 'UPDATING' test.done(); }, @@ -386,20 +445,13 @@ export = { version: '1.1', })); - let count = 3; - const restore = mocks.client.describeCluster; - mocks.client.describeCluster = async () => ({ - cluster: { status: --count === 0 ? 'UPDATING' : 'ACTIVE' }, - }); const resp = await handler.onEvent(); - mocks.client.describeCluster = restore; - test.equal(resp, undefined); + test.deepEqual(resp, { EksUpdateId: mocks.MOCK_UPDATE_STATUS_ID }); test.deepEqual(mocks.actualRequest.updateClusterVersionRequest!, { name: 'physical-resource-id', version: '2.0', }); test.equal(mocks.actualRequest.createClusterRequest, undefined); - test.equal(count, 0); // make sure "describeCluster" was called until it returned 'UPDATING' test.done(); }, diff --git a/packages/@aws-cdk/aws-eks/test/test.fargate-resource-provider.ts b/packages/@aws-cdk/aws-eks/test/test.fargate-resource-provider.ts index 873b1b499cdbe..5c119ae7a9c4e 100644 --- a/packages/@aws-cdk/aws-eks/test/test.fargate-resource-provider.ts +++ b/packages/@aws-cdk/aws-eks/test/test.fargate-resource-provider.ts @@ -287,6 +287,7 @@ function newEksClientMock() { createCluster: sinon.fake.throws('not implemented'), deleteCluster: sinon.fake.throws('not implemented'), describeCluster: sinon.fake.throws('not implemented'), + describeUpdate: sinon.fake.throws('not implemented'), updateClusterConfig: sinon.fake.throws('not implemented'), updateClusterVersion: sinon.fake.throws('not implemented'), configureAssumeRole: sinon.fake(), From 074a7d28b0e5d24fb6bf47bd5ecfbc6cf3814e32 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 23 Apr 2020 15:20:23 +0300 Subject: [PATCH 5/6] update timeout back to 1m --- .../aws-eks/lib/cluster-resource-provider.ts | 2 +- .../test/integ.eks-cluster.expected.json | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts index e7dfe30ed1522..a3e3128f807b8 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource-provider.ts @@ -41,7 +41,7 @@ export class ClusterResourceProvider extends NestedStack { description: 'onEvent handler for EKS cluster resource provider', runtime: HANDLER_RUNTIME, handler: 'index.onEvent', - timeout: Duration.minutes(15), + timeout: Duration.minutes(1), }); const isComplete = new lambda.Function(this, 'IsCompleteHandler', { diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index 6d59deb5a14a9..79a3ce4e10f40 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -2231,7 +2231,7 @@ }, "/", { - "Ref": "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3Bucket889DD28D" + "Ref": "AssetParametersfa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103S3Bucket4281E0A4" }, "/", { @@ -2241,7 +2241,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3VersionKey9765793B" + "Ref": "AssetParametersfa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103S3VersionKey3B54BD32" } ] } @@ -2254,7 +2254,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3VersionKey9765793B" + "Ref": "AssetParametersfa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103S3VersionKey3B54BD32" } ] } @@ -2449,17 +2449,17 @@ "Type": "String", "Description": "Artifact hash for asset \"a6d508eaaa0d3cddbb47a84123fc878809c8431c5466f360912f70b5b9770afb\"" }, - "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3Bucket889DD28D": { + "AssetParametersfa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103S3Bucket4281E0A4": { "Type": "String", - "Description": "S3 bucket for asset \"41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16e\"" + "Description": "S3 bucket for asset \"fa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103\"" }, - "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eS3VersionKey9765793B": { + "AssetParametersfa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103S3VersionKey3B54BD32": { "Type": "String", - "Description": "S3 key for asset version \"41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16e\"" + "Description": "S3 key for asset version \"fa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103\"" }, - "AssetParameters41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16eArtifactHash1A2B5996": { + "AssetParametersfa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103ArtifactHash733CC5DF": { "Type": "String", - "Description": "Artifact hash for asset \"41394e7867caf11d6e9496a67d25a48ef824d22cd8e8cf1d39c7d4f27c41d16e\"" + "Description": "Artifact hash for asset \"fa73027e9f72f21daca2d67aa5a23e88f87d90536a7e3c36de9adbfb27fa9103\"" }, "AssetParameters36525a61abfaf5764fad460fd03c24215fd00da60805807d6138c51be4d03dbcS3Bucket2D824DEF": { "Type": "String", From 64633449532a3f1c59faf61df76636457f5ae8f8 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Thu, 23 Apr 2020 16:51:17 +0300 Subject: [PATCH 6/6] add DescribeUpdate permission --- packages/@aws-cdk/aws-eks/lib/cluster-resource.ts | 1 + packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json | 1 + packages/@aws-cdk/aws-eks/test/test.cluster.ts | 2 ++ 3 files changed, 4 insertions(+) diff --git a/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts b/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts index e25198d5aa8d5..c449af08a407b 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster-resource.ts @@ -81,6 +81,7 @@ export class ClusterResource extends Construct { actions: [ 'eks:CreateCluster', 'eks:DescribeCluster', + 'eks:DescribeUpdate', 'eks:DeleteCluster', 'eks:UpdateClusterVersion', 'eks:UpdateClusterConfig', diff --git a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json index 79a3ce4e10f40..c168d90725028 100644 --- a/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json +++ b/packages/@aws-cdk/aws-eks/test/integ.eks-cluster.expected.json @@ -802,6 +802,7 @@ "Action": [ "eks:CreateCluster", "eks:DescribeCluster", + "eks:DescribeUpdate", "eks:DeleteCluster", "eks:UpdateClusterVersion", "eks:UpdateClusterConfig", diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster.ts b/packages/@aws-cdk/aws-eks/test/test.cluster.ts index eb2007c70e627..7e67d9f0c8632 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster.ts @@ -831,6 +831,7 @@ export = { Action: [ 'eks:CreateCluster', 'eks:DescribeCluster', + 'eks:DescribeUpdate', 'eks:DeleteCluster', 'eks:UpdateClusterVersion', 'eks:UpdateClusterConfig', @@ -941,6 +942,7 @@ export = { Action: [ 'eks:CreateCluster', 'eks:DescribeCluster', + 'eks:DescribeUpdate', 'eks:DeleteCluster', 'eks:UpdateClusterVersion', 'eks:UpdateClusterConfig',