From 0aa7adac958fb7997b64eba8c7fc3008e8557480 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Wed, 2 Sep 2020 17:24:35 +0300 Subject: [PATCH] feat(eks): kubectl layer customization (#10090) Allow providing a custom kubectl lambda layer and/or creating the KubectlLayer object with custom configuration (version, aped). Resolves #7992 ---- *By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license* --- packages/@aws-cdk/aws-eks/README.md | 41 ++++++++++- packages/@aws-cdk/aws-eks/lib/cluster.ts | 71 +++++++++++++++++++ packages/@aws-cdk/aws-eks/lib/index.ts | 3 +- .../@aws-cdk/aws-eks/lib/kubectl-layer.ts | 50 +++++++------ .../@aws-cdk/aws-eks/lib/kubectl-provider.ts | 24 ++++++- .../@aws-cdk/aws-eks/test/test.cluster.ts | 62 ++++++++++++++-- 6 files changed, 217 insertions(+), 34 deletions(-) diff --git a/packages/@aws-cdk/aws-eks/README.md b/packages/@aws-cdk/aws-eks/README.md index 605666e11ac2e..59844cf4c179e 100644 --- a/packages/@aws-cdk/aws-eks/README.md +++ b/packages/@aws-cdk/aws-eks/README.md @@ -322,7 +322,7 @@ new KubernetesManifest(this, 'hello-kub', { cluster.addManifest('hello-kub', service, deployment); ``` -#### Kubectl Environment +#### Kubectl Layer and Environment The resources are created in the cluster by running `kubectl apply` from a python lambda function. You can configure the environment of this function by specifying it at cluster instantiation. For example, this can useful in order to configure an http proxy: @@ -335,6 +335,45 @@ const cluster = new eks.Cluster(this, 'hello-eks', { }); ``` +By default, the `kubectl`, `helm` and `aws` commands used to operate the cluster +are provided by an AWS Lambda Layer from the AWS Serverless Application +in [aws-lambda-layer-kubectl]. In most cases this should be sufficient. + +You can provide a custom layer in case the default layer does not meet your +needs or if the SAR app is not available in your region. + +```ts +// custom build: +const layer = new lambda.LayerVersion(this, 'KubectlLayer', { + code: lambda.Code.fromAsset(`${__dirname}/layer.zip`)), + compatibleRuntimes: [lambda.Runtime.PROVIDED] +}); + +// or, a specific version or appid of aws-lambda-layer-kubectl: +const layer = new eks.KubectlLayer(this, 'KubectlLayer', { + version: '2.0.0', // optional + applicationId: '...' // optional +}); +``` + +Pass it to `kubectlLayer` when you create or import a cluster: + +```ts +const cluster = new eks.Cluster(this, 'MyCluster', { + kubectlLayer: layer, +}); + +// or +const cluster = eks.Cluster.fromClusterAttributes(this, 'MyCluster', { + kubectlLayer: layer, +}); +``` + +> Instructions on how to build `layer.zip` can be found +> [here](https://github.com/aws-samples/aws-lambda-layer-kubectl/blob/master/cdk/README.md). + +[aws-lambda-layer-kubectl]: https://github.com/aws-samples/aws-lambda-layer-kubectl + #### Adding resources from a URL The following example will deploy the resource manifest hosting on remote server: diff --git a/packages/@aws-cdk/aws-eks/lib/cluster.ts b/packages/@aws-cdk/aws-eks/lib/cluster.ts index 8e7f060973694..3889ea41d6d32 100644 --- a/packages/@aws-cdk/aws-eks/lib/cluster.ts +++ b/packages/@aws-cdk/aws-eks/lib/cluster.ts @@ -4,6 +4,7 @@ import * as autoscaling from '@aws-cdk/aws-autoscaling'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; +import * as lambda from '@aws-cdk/aws-lambda'; import * as ssm from '@aws-cdk/aws-ssm'; import { CfnOutput, CfnResource, Construct, IResource, Resource, Stack, Tags, Token, Duration } from '@aws-cdk/core'; import * as YAML from 'yaml'; @@ -98,6 +99,13 @@ export interface ICluster extends IResource, ec2.IConnectable { */ readonly kubectlPrivateSubnets?: ec2.ISubnet[]; + /** + * An AWS Lambda layer that includes `kubectl`, `helm` and the `aws` CLI. + * + * If not defined, a default layer will be used. + */ + readonly kubectlLayer?: lambda.ILayerVersion; + /** * Defines a Kubernetes resource in this cluster. * @@ -194,6 +202,36 @@ export interface ClusterAttributes { * @default - k8s endpoint is expected to be accessible publicly */ readonly kubectlPrivateSubnetIds?: string[]; + + /** + * An AWS Lambda Layer which includes `kubectl`, Helm and the AWS CLI. + * + * By default, the provider will use the layer included in the + * "aws-lambda-layer-kubectl" SAR application which is available in all + * commercial regions. + * + * To deploy the layer locally, visit + * https://github.com/aws-samples/aws-lambda-layer-kubectl/blob/master/cdk/README.md + * for instructions on how to prepare the .zip file and then define it in your + * app as follows: + * + * ```ts + * const layer = new lambda.LayerVersion(this, 'kubectl-layer', { + * code: lambda.Code.fromAsset(`${__dirname}/layer.zip`)), + * compatibleRuntimes: [lambda.Runtime.PROVIDED] + * }); + * + * Or you can use the standard layer like this (with options + * to customize the version and SAR application ID): + * + * ```ts + * const layer = new eks.KubectlLayer(this, 'KubectlLayer'); + * ``` + * + * @default - the layer provided by the `aws-lambda-layer-kubectl` SAR app. + * @see https://github.com/aws-samples/aws-lambda-layer-kubectl + */ + readonly kubectlLayer?: lambda.ILayerVersion; } /** @@ -315,6 +353,30 @@ export interface ClusterOptions extends CommonClusterOptions { * @default - No environment variables. */ readonly kubectlEnvironment?: { [key: string]: string }; + + /** + * An AWS Lambda Layer which includes `kubectl`, Helm and the AWS CLI. + * + * By default, the provider will use the layer included in the + * "aws-lambda-layer-kubectl" SAR application which is available in all + * commercial regions. + * + * To deploy the layer locally, visit + * https://github.com/aws-samples/aws-lambda-layer-kubectl/blob/master/cdk/README.md + * for instructions on how to prepare the .zip file and then define it in your + * app as follows: + * + * ```ts + * const layer = new lambda.LayerVersion(this, 'kubectl-layer', { + * code: lambda.Code.fromAsset(`${__dirname}/layer.zip`)), + * compatibleRuntimes: [lambda.Runtime.PROVIDED] + * }) + * ``` + * + * @default - the layer provided by the `aws-lambda-layer-kubectl` SAR app. + * @see https://github.com/aws-samples/aws-lambda-layer-kubectl + */ + readonly kubectlLayer?: lambda.ILayerVersion; } /** @@ -693,6 +755,12 @@ export class Cluster extends ClusterBase { */ private readonly _fargateProfiles: FargateProfile[] = []; + /** + * The AWS Lambda layer that contains `kubectl`, `helm` and the AWS CLI. If + * undefined, a SAR app that contains this layer will be used. + */ + public readonly kubectlLayer?: lambda.ILayerVersion; + /** * If this cluster is kubectl-enabled, returns the `ClusterResource` object * that manages it. If this cluster is not kubectl-enabled (i.e. uses the @@ -786,6 +854,7 @@ export class Cluster extends ClusterBase { this.endpointAccess = props.endpointAccess ?? EndpointAccess.PUBLIC_AND_PRIVATE; this.kubectlEnvironment = props.kubectlEnvironment; + this.kubectlLayer = props.kubectlLayer; if (this.endpointAccess._config.privateAccess && this.vpc instanceof ec2.Vpc) { // validate VPC properties according to: https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html @@ -1482,6 +1551,7 @@ class ImportedCluster extends ClusterBase implements ICluster { public readonly kubectlEnvironment?: { [key: string]: string; } | undefined; public readonly kubectlSecurityGroup?: ec2.ISecurityGroup | undefined; public readonly kubectlPrivateSubnets?: ec2.ISubnet[] | undefined; + public readonly kubectlLayer?: lambda.ILayerVersion; constructor(scope: Construct, id: string, private readonly props: ClusterAttributes) { super(scope, id); @@ -1492,6 +1562,7 @@ class ImportedCluster extends ClusterBase implements ICluster { this.kubectlSecurityGroup = props.kubectlSecurityGroupId ? ec2.SecurityGroup.fromSecurityGroupId(this, 'KubectlSecurityGroup', props.kubectlSecurityGroupId) : undefined; this.kubectlEnvironment = props.kubectlEnvironment; this.kubectlPrivateSubnets = props.kubectlPrivateSubnetIds ? props.kubectlPrivateSubnetIds.map(subnetid => ec2.Subnet.fromSubnetId(this, `KubectlSubnet${subnetid}`, subnetid)) : undefined; + this.kubectlLayer = props.kubectlLayer; let i = 1; for (const sgid of props.securityGroupIds ?? []) { diff --git a/packages/@aws-cdk/aws-eks/lib/index.ts b/packages/@aws-cdk/aws-eks/lib/index.ts index e1d353959096f..2e4b7e47ae49c 100644 --- a/packages/@aws-cdk/aws-eks/lib/index.ts +++ b/packages/@aws-cdk/aws-eks/lib/index.ts @@ -10,4 +10,5 @@ export * from './k8s-manifest'; export * from './k8s-object-value'; export * from './fargate-cluster'; export * from './service-account'; -export * from './managed-nodegroup'; \ No newline at end of file +export * from './managed-nodegroup'; +export * from './kubectl-layer'; \ No newline at end of file diff --git a/packages/@aws-cdk/aws-eks/lib/kubectl-layer.ts b/packages/@aws-cdk/aws-eks/lib/kubectl-layer.ts index 6112d0b81765b..5221f1b1c279c 100644 --- a/packages/@aws-cdk/aws-eks/lib/kubectl-layer.ts +++ b/packages/@aws-cdk/aws-eks/lib/kubectl-layer.ts @@ -1,19 +1,28 @@ import * as crypto from 'crypto'; import * as lambda from '@aws-cdk/aws-lambda'; -import { CfnResource, Construct, Resource, Stack, Token } from '@aws-cdk/core'; +import { CfnResource, Construct, Token, Stack, ResourceEnvironment } from '@aws-cdk/core'; const KUBECTL_APP_ARN = 'arn:aws:serverlessrepo:us-east-1:903779448426:applications/lambda-layer-kubectl'; const KUBECTL_APP_CN_ARN = 'arn:aws-cn:serverlessrepo:cn-north-1:487369736442:applications/lambda-layer-kubectl'; +const KUBECTL_APP_VERSION = '2.0.0'; -const KUBECTL_APP_VERSION = '1.13.7'; - +/** + * Properties for KubectlLayer. + */ export interface KubectlLayerProps { /** * The semantic version of the kubectl AWS Lambda Layer SAR app to use. * - * @default '1.13.7' + * @default '2.0.0' */ readonly version?: string; + + /** + * The Serverless Application Repository application ID which contains the kubectl layer. + * @default - The ARN for the `lambda-layer-kubectl` SAR app. + * @see https://github.com/aws-samples/aws-lambda-layer-kubectl + */ + readonly applicationId?: string; } /** @@ -21,27 +30,15 @@ export interface KubectlLayerProps { * * @see https://github.com/aws-samples/aws-lambda-layer-kubectl */ -export class KubectlLayer extends Resource implements lambda.ILayerVersion { - - /** - * Gets or create a singleton instance of this construct. - */ - public static getOrCreate(scope: Construct, props: KubectlLayerProps = {}): KubectlLayer { - const stack = Stack.of(scope); - const id = 'kubectl-layer-' + (props.version ? props.version : '8C2542BC-BF2B-4DFE-B765-E181FD30A9A0'); - const exists = stack.node.tryFindChild(id) as KubectlLayer; - if (exists) { - return exists; - } - - return new KubectlLayer(stack, id, props); - } - +export class KubectlLayer extends Construct implements lambda.ILayerVersion { /** * The ARN of the AWS Lambda layer version. */ public readonly layerVersionArn: string; + public readonly stack: Stack; + public readonly env: ResourceEnvironment; + /** * All runtimes are compatible. */ @@ -50,15 +47,22 @@ export class KubectlLayer extends Resource implements lambda.ILayerVersion { constructor(scope: Construct, id: string, props: KubectlLayerProps = {}) { super(scope, id); + this.stack = Stack.of(this); + this.env = { + account: this.stack.account, + region: this.stack.region, + }; + const uniqueId = crypto.createHash('md5').update(this.node.path).digest('hex'); - const version = props.version || KUBECTL_APP_VERSION; + const version = props.version ?? KUBECTL_APP_VERSION; + const applictionId = props.applicationId ?? (this.isChina() ? KUBECTL_APP_CN_ARN : KUBECTL_APP_ARN); this.stack.templateOptions.transforms = ['AWS::Serverless-2016-10-31']; // required for AWS::Serverless const resource = new CfnResource(this, 'Resource', { type: 'AWS::Serverless::Application', properties: { Location: { - ApplicationId: this.isChina() ? KUBECTL_APP_CN_ARN : KUBECTL_APP_ARN, + ApplicationId: applictionId, SemanticVersion: version, }, Parameters: { @@ -74,7 +78,7 @@ export class KubectlLayer extends Resource implements lambda.ILayerVersion { return; } - public isChina(): boolean { + private isChina(): boolean { const region = this.stack.region; return !Token.isUnresolved(region) && region.startsWith('cn-'); } diff --git a/packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts b/packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts index 6af14cbce4a16..bdbfbbc54f6fc 100644 --- a/packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts +++ b/packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts @@ -4,7 +4,7 @@ import * as lambda from '@aws-cdk/aws-lambda'; import { Construct, Duration, Stack, NestedStack } from '@aws-cdk/core'; import * as cr from '@aws-cdk/custom-resources'; import { ICluster, Cluster } from './cluster'; -import { KubectlLayer } from './kubectl-layer'; +import { KubectlLayer, KubectlLayerProps } from './kubectl-layer'; export interface KubectlProviderProps { /** @@ -14,8 +14,8 @@ export interface KubectlProviderProps { } export class KubectlProvider extends NestedStack { - public static getOrCreate(scope: Construct, cluster: ICluster) { + public static getOrCreate(scope: Construct, cluster: ICluster) { // if this is an "owned" cluster, it has a provider associated with it if (cluster instanceof Cluster) { return cluster._attachKubectlResourceScope(scope); @@ -61,13 +61,15 @@ export class KubectlProvider extends NestedStack { throw new Error('"kubectlSecurityGroup" is required if "kubectlSubnets" is specified'); } + const layer = cluster.kubectlLayer ?? getOrCreateKubectlLayer(this); + const handler = new lambda.Function(this, 'Handler', { code: lambda.Code.fromAsset(path.join(__dirname, 'kubectl-handler')), runtime: lambda.Runtime.PYTHON_3_7, handler: 'index.handler', timeout: Duration.minutes(15), description: 'onEvent handler for EKS kubectl resource provider', - layers: [KubectlLayer.getOrCreate(this, { version: '2.0.0' })], + layers: [layer], memorySize: 256, environment: cluster.kubectlEnvironment, @@ -94,5 +96,21 @@ export class KubectlProvider extends NestedStack { this.serviceToken = provider.serviceToken; this.roleArn = cluster.kubectlRole.roleArn; } + } +/** + * Gets or create a singleton instance of KubectlLayer. + * + * (exported for unit tests). + */ +export function getOrCreateKubectlLayer(scope: Construct, props: KubectlLayerProps = {}): KubectlLayer { + const stack = Stack.of(scope); + const id = 'kubectl-layer-' + (props.version ? props.version : '8C2542BC-BF2B-4DFE-B765-E181FD30A9A0'); + const exists = stack.node.tryFindChild(id) as KubectlLayer; + if (exists) { + return exists; + } + + return new KubectlLayer(stack, id, props); +} diff --git a/packages/@aws-cdk/aws-eks/test/test.cluster.ts b/packages/@aws-cdk/aws-eks/test/test.cluster.ts index e8bd1e3f70b09..7ecd86cb269d1 100644 --- a/packages/@aws-cdk/aws-eks/test/test.cluster.ts +++ b/packages/@aws-cdk/aws-eks/test/test.cluster.ts @@ -5,11 +5,12 @@ import * as asg from '@aws-cdk/aws-autoscaling'; import * as ec2 from '@aws-cdk/aws-ec2'; import * as iam from '@aws-cdk/aws-iam'; import * as kms from '@aws-cdk/aws-kms'; +import * as lambda from '@aws-cdk/aws-lambda'; import * as cdk from '@aws-cdk/core'; import { Test } from 'nodeunit'; import * as YAML from 'yaml'; import * as eks from '../lib'; -import { KubectlLayer } from '../lib/kubectl-layer'; +import { getOrCreateKubectlLayer } from '../lib/kubectl-provider'; import { testFixture, testFixtureNoVpc } from './util'; /* eslint-disable max-len */ @@ -257,7 +258,7 @@ export = { // WHEN const vpc = new ec2.Vpc(stack, 'VPC'); new eks.Cluster(stack, 'Cluster', { vpc, defaultCapacity: 0, version: CLUSTER_VERSION }); - const layer = KubectlLayer.getOrCreate(stack, {}); + getOrCreateKubectlLayer(stack); // THEN expect(stack).to(haveResource('Custom::AWSCDK-EKS-Cluster')); @@ -266,7 +267,6 @@ export = { ApplicationId: 'arn:aws:serverlessrepo:us-east-1:903779448426:applications/lambda-layer-kubectl', }, })); - test.equal(layer.isChina(), false); test.done(); }, @@ -278,8 +278,7 @@ export = { // WHEN const vpc = new ec2.Vpc(stack, 'VPC'); new eks.Cluster(stack, 'Cluster', { vpc, defaultCapacity: 0, version: CLUSTER_VERSION }); - new KubectlLayer(stack, 'NewLayer'); - const layer = KubectlLayer.getOrCreate(stack); + getOrCreateKubectlLayer(stack); // THEN expect(stack).to(haveResource('Custom::AWSCDK-EKS-Cluster')); @@ -288,7 +287,6 @@ export = { ApplicationId: 'arn:aws-cn:serverlessrepo:cn-north-1:487369736442:applications/lambda-layer-kubectl', }, })); - test.equal(layer.isChina(), true); test.done(); }, @@ -2003,6 +2001,58 @@ export = { test.deepEqual(rawTemplate.Outputs.LoadBalancerAddress.Value, { 'Fn::GetAtt': [expectedKubernetesGetId, 'Value'] }); test.done(); }, + + 'custom kubectl layer can be provided'(test: Test) { + // GIVEN + const { stack } = testFixture(); + + // WHEN + const layer = lambda.LayerVersion.fromLayerVersionArn(stack, 'MyLayer', 'arn:of:layer'); + new eks.Cluster(stack, 'Cluster1', { + version: CLUSTER_VERSION, + kubectlLayer: layer, + }); + + // THEN + const providerStack = stack.node.tryFindChild('@aws-cdk/aws-eks.KubectlProvider') as cdk.NestedStack; + expect(providerStack).to(haveResource('AWS::Lambda::Function', { + Layers: ['arn:of:layer'], + })); + + test.done(); + }, + + 'SAR-based kubectl layer can be customized'(test: Test) { + // GIVEN + const { stack } = testFixture(); + + // WHEN + const layer = new eks.KubectlLayer(stack, 'Kubectl', { + applicationId: 'custom:app:id', + version: '2.3.4', + }); + + new eks.Cluster(stack, 'Cluster1', { + version: CLUSTER_VERSION, + kubectlLayer: layer, + }); + + // THEN + const providerStack = stack.node.tryFindChild('@aws-cdk/aws-eks.KubectlProvider') as cdk.NestedStack; + expect(providerStack).to(haveResource('AWS::Lambda::Function', { + Layers: [{ Ref: 'referencetoStackKubectl7F29063EOutputsLayerVersionArn' }], + })); + + expect(stack).to(haveResource('AWS::Serverless::Application', { + Location: { + ApplicationId: 'custom:app:id', + SemanticVersion: '2.3.4', + }, + })); + + test.done(); + }, + 'create a cluster using custom resource with secrets encryption using KMS CMK'(test: Test) { // GIVEN const { stack, vpc } = testFixture();