feat(eks): kubectl layer customization (#10090)

Allow providing a custom kubectl lambda layer and/or creating the KubectlLayer object with custom configuration (version, aped).

Resolves #7992

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*

packages/@aws-cdk/aws-eks/README.md

@@ -322,7 +322,7 @@ new KubernetesManifest(this, 'hello-kub', {
 cluster.addManifest('hello-kub', service, deployment);
 ```
 
-#### Kubectl Environment
+#### Kubectl Layer and Environment
 
 The resources are created in the cluster by running `kubectl apply` from a python lambda function. You can configure the environment of this function by specifying it at cluster instantiation. For example, this can useful in order to configure an http proxy:
 
@@ -335,6 +335,45 @@ const cluster = new eks.Cluster(this, 'hello-eks', {
 });
 ```
 
+By default, the `kubectl`, `helm` and `aws` commands used to operate the cluster
+are provided by an AWS Lambda Layer from the AWS Serverless Application
+in [aws-lambda-layer-kubectl]. In most cases this should be sufficient.
+
+You can provide a custom layer in case the default layer does not meet your
+needs or if the SAR app is not available in your region.
+
+```ts
+// custom build:
+const layer = new lambda.LayerVersion(this, 'KubectlLayer', {
+  code: lambda.Code.fromAsset(`${__dirname}/layer.zip`)),
+  compatibleRuntimes: [lambda.Runtime.PROVIDED]
+});
+
+// or, a specific version or appid of aws-lambda-layer-kubectl:
+const layer = new eks.KubectlLayer(this, 'KubectlLayer', {
+  version: '2.0.0',    // optional
+  applicationId: '...' // optional
+});
+```
+
+Pass it to `kubectlLayer` when you create or import a cluster:
+
+```ts
+const cluster = new eks.Cluster(this, 'MyCluster', {
+  kubectlLayer: layer,
+});
+
+// or
+const cluster = eks.Cluster.fromClusterAttributes(this, 'MyCluster', {
+  kubectlLayer: layer,
+});
+```
+
+> Instructions on how to build `layer.zip` can be found
+> [here](https://github.com/aws-samples/aws-lambda-layer-kubectl/blob/master/cdk/README.md).
+
+[aws-lambda-layer-kubectl]: https://github.com/aws-samples/aws-lambda-layer-kubectl
+
 #### Adding resources from a URL
 
 The following example will deploy the resource manifest hosting on remote server:

packages/@aws-cdk/aws-eks/lib/cluster.ts

@@ -4,6 +4,7 @@ import * as autoscaling from '@aws-cdk/aws-autoscaling';
 import * as ec2 from '@aws-cdk/aws-ec2';
 import * as iam from '@aws-cdk/aws-iam';
 import * as kms from '@aws-cdk/aws-kms';
+import * as lambda from '@aws-cdk/aws-lambda';
 import * as ssm from '@aws-cdk/aws-ssm';
 import { CfnOutput, CfnResource, Construct, IResource, Resource, Stack, Tags, Token, Duration } from '@aws-cdk/core';
 import * as YAML from 'yaml';
@@ -98,6 +99,13 @@ export interface ICluster extends IResource, ec2.IConnectable {
    */
   readonly kubectlPrivateSubnets?: ec2.ISubnet[];
 
+  /**
+   * An AWS Lambda layer that includes `kubectl`, `helm` and the `aws` CLI.
+   *
+   * If not defined, a default layer will be used.
+   */
+  readonly kubectlLayer?: lambda.ILayerVersion;
+
   /**
    * Defines a Kubernetes resource in this cluster.
    *
@@ -194,6 +202,36 @@ export interface ClusterAttributes {
    * @default - k8s endpoint is expected to be accessible publicly
    */
   readonly kubectlPrivateSubnetIds?: string[];
+
+  /**
+   * An AWS Lambda Layer which includes `kubectl`, Helm and the AWS CLI.
+   *
+   * By default, the provider will use the layer included in the
+   * "aws-lambda-layer-kubectl" SAR application which is available in all
+   * commercial regions.
+   *
+   * To deploy the layer locally, visit
+   * https://github.com/aws-samples/aws-lambda-layer-kubectl/blob/master/cdk/README.md
+   * for instructions on how to prepare the .zip file and then define it in your
+   * app as follows:
+   *
+   * ```ts
+   * const layer = new lambda.LayerVersion(this, 'kubectl-layer', {
+   *   code: lambda.Code.fromAsset(`${__dirname}/layer.zip`)),
+   *   compatibleRuntimes: [lambda.Runtime.PROVIDED]
+   * });
+   *
+   * Or you can use the standard layer like this (with options
+   * to customize the version and SAR application ID):
+   *
+   * ```ts
+   * const layer = new eks.KubectlLayer(this, 'KubectlLayer');
+   * ```
+   *
+   * @default - the layer provided by the `aws-lambda-layer-kubectl` SAR app.
+   * @see https://github.com/aws-samples/aws-lambda-layer-kubectl
+   */
+  readonly kubectlLayer?: lambda.ILayerVersion;
 }
 
 /**
@@ -315,6 +353,30 @@ export interface ClusterOptions extends CommonClusterOptions {
    * @default - No environment variables.
    */
   readonly kubectlEnvironment?: { [key: string]: string };
+
+  /**
+   * An AWS Lambda Layer which includes `kubectl`, Helm and the AWS CLI.
+   *
+   * By default, the provider will use the layer included in the
+   * "aws-lambda-layer-kubectl" SAR application which is available in all
+   * commercial regions.
+   *
+   * To deploy the layer locally, visit
+   * https://github.com/aws-samples/aws-lambda-layer-kubectl/blob/master/cdk/README.md
+   * for instructions on how to prepare the .zip file and then define it in your
+   * app as follows:
+   *
+   * ```ts
+   * const layer = new lambda.LayerVersion(this, 'kubectl-layer', {
+   *   code: lambda.Code.fromAsset(`${__dirname}/layer.zip`)),
+   *   compatibleRuntimes: [lambda.Runtime.PROVIDED]
+   * })
+   * ```
+   *
+   * @default - the layer provided by the `aws-lambda-layer-kubectl` SAR app.
+   * @see https://github.com/aws-samples/aws-lambda-layer-kubectl
+   */
+  readonly kubectlLayer?: lambda.ILayerVersion;
 }
 
 /**
@@ -693,6 +755,12 @@ export class Cluster extends ClusterBase {
    */
   private readonly _fargateProfiles: FargateProfile[] = [];
 
+  /**
+   * The AWS Lambda layer that contains `kubectl`, `helm` and the AWS CLI. If
+   * undefined, a SAR app that contains this layer will be used.
+   */
+  public readonly kubectlLayer?: lambda.ILayerVersion;
+
   /**
    * If this cluster is kubectl-enabled, returns the `ClusterResource` object
    * that manages it. If this cluster is not kubectl-enabled (i.e. uses the
@@ -786,6 +854,7 @@ export class Cluster extends ClusterBase {
 
     this.endpointAccess = props.endpointAccess ?? EndpointAccess.PUBLIC_AND_PRIVATE;
     this.kubectlEnvironment = props.kubectlEnvironment;
+    this.kubectlLayer = props.kubectlLayer;
 
     if (this.endpointAccess._config.privateAccess && this.vpc instanceof ec2.Vpc) {
       // validate VPC properties according to: https://docs.aws.amazon.com/eks/latest/userguide/cluster-endpoint.html
@@ -1482,6 +1551,7 @@ class ImportedCluster extends ClusterBase implements ICluster {
   public readonly kubectlEnvironment?: { [key: string]: string; } | undefined;
   public readonly kubectlSecurityGroup?: ec2.ISecurityGroup | undefined;
   public readonly kubectlPrivateSubnets?: ec2.ISubnet[] | undefined;
+  public readonly kubectlLayer?: lambda.ILayerVersion;
 
   constructor(scope: Construct, id: string, private readonly props: ClusterAttributes) {
     super(scope, id);
@@ -1492,6 +1562,7 @@ class ImportedCluster extends ClusterBase implements ICluster {
     this.kubectlSecurityGroup = props.kubectlSecurityGroupId ? ec2.SecurityGroup.fromSecurityGroupId(this, 'KubectlSecurityGroup', props.kubectlSecurityGroupId) : undefined;
     this.kubectlEnvironment = props.kubectlEnvironment;
     this.kubectlPrivateSubnets = props.kubectlPrivateSubnetIds ? props.kubectlPrivateSubnetIds.map(subnetid => ec2.Subnet.fromSubnetId(this, `KubectlSubnet${subnetid}`, subnetid)) : undefined;
+    this.kubectlLayer = props.kubectlLayer;
 
     let i = 1;
     for (const sgid of props.securityGroupIds ?? []) {

packages/@aws-cdk/aws-eks/lib/index.ts

@@ -10,4 +10,5 @@ export * from './k8s-manifest';
 export * from './k8s-object-value';
 export * from './fargate-cluster';
 export * from './service-account';
-export * from './managed-nodegroup';
+export * from './managed-nodegroup';
+export * from './kubectl-layer';

packages/@aws-cdk/aws-eks/lib/kubectl-layer.ts

@@ -1,47 +1,44 @@
 import * as crypto from 'crypto';
 import * as lambda from '@aws-cdk/aws-lambda';
-import { CfnResource, Construct, Resource, Stack, Token } from '@aws-cdk/core';
+import { CfnResource, Construct, Token, Stack, ResourceEnvironment } from '@aws-cdk/core';
 
 const KUBECTL_APP_ARN = 'arn:aws:serverlessrepo:us-east-1:903779448426:applications/lambda-layer-kubectl';
 const KUBECTL_APP_CN_ARN = 'arn:aws-cn:serverlessrepo:cn-north-1:487369736442:applications/lambda-layer-kubectl';
+const KUBECTL_APP_VERSION = '2.0.0';
 
-const KUBECTL_APP_VERSION = '1.13.7';
-
+/**
+ * Properties for KubectlLayer.
+ */
 export interface KubectlLayerProps {
   /**
    * The semantic version of the kubectl AWS Lambda Layer SAR app to use.
    *
-   * @default '1.13.7'
+   * @default '2.0.0'
    */
   readonly version?: string;
+
+  /**
+   * The Serverless Application Repository application ID which contains the kubectl layer.
+   * @default - The ARN for the `lambda-layer-kubectl` SAR app.
+   * @see https://github.com/aws-samples/aws-lambda-layer-kubectl
+   */
+  readonly applicationId?: string;
 }
 
 /**
  * An AWS Lambda layer that includes kubectl and the AWS CLI.
  *
  * @see https://github.com/aws-samples/aws-lambda-layer-kubectl
  */
-export class KubectlLayer extends Resource implements lambda.ILayerVersion {
-
-  /**
-   * Gets or create a singleton instance of this construct.
-   */
-  public static getOrCreate(scope: Construct, props: KubectlLayerProps = {}): KubectlLayer {
-    const stack = Stack.of(scope);
-    const id = 'kubectl-layer-' + (props.version ? props.version : '8C2542BC-BF2B-4DFE-B765-E181FD30A9A0');
-    const exists = stack.node.tryFindChild(id) as KubectlLayer;
-    if (exists) {
-      return exists;
-    }
-
-    return new KubectlLayer(stack, id, props);
-  }
-
+export class KubectlLayer extends Construct implements lambda.ILayerVersion {
   /**
    * The ARN of the AWS Lambda layer version.
    */
   public readonly layerVersionArn: string;
 
+  public readonly stack: Stack;
+  public readonly env: ResourceEnvironment;
+
   /**
    * All runtimes are compatible.
    */
@@ -50,15 +47,22 @@ export class KubectlLayer extends Resource implements lambda.ILayerVersion {
   constructor(scope: Construct, id: string, props: KubectlLayerProps = {}) {
     super(scope, id);
 
+    this.stack = Stack.of(this);
+    this.env = {
+      account: this.stack.account,
+      region: this.stack.region,
+    };
+
     const uniqueId = crypto.createHash('md5').update(this.node.path).digest('hex');
-    const version = props.version || KUBECTL_APP_VERSION;
+    const version = props.version ?? KUBECTL_APP_VERSION;
+    const applictionId = props.applicationId ?? (this.isChina() ? KUBECTL_APP_CN_ARN : KUBECTL_APP_ARN);
 
     this.stack.templateOptions.transforms = ['AWS::Serverless-2016-10-31']; // required for AWS::Serverless
     const resource = new CfnResource(this, 'Resource', {
       type: 'AWS::Serverless::Application',
       properties: {
         Location: {
-          ApplicationId: this.isChina() ? KUBECTL_APP_CN_ARN : KUBECTL_APP_ARN,
+          ApplicationId: applictionId,
           SemanticVersion: version,
         },
         Parameters: {
@@ -74,7 +78,7 @@ export class KubectlLayer extends Resource implements lambda.ILayerVersion {
     return;
   }
 
-  public isChina(): boolean {
+  private isChina(): boolean {
     const region = this.stack.region;
     return !Token.isUnresolved(region) && region.startsWith('cn-');
   }

packages/@aws-cdk/aws-eks/lib/kubectl-provider.ts

@@ -4,7 +4,7 @@ import * as lambda from '@aws-cdk/aws-lambda';
 import { Construct, Duration, Stack, NestedStack } from '@aws-cdk/core';
 import * as cr from '@aws-cdk/custom-resources';
 import { ICluster, Cluster } from './cluster';
-import { KubectlLayer } from './kubectl-layer';
+import { KubectlLayer, KubectlLayerProps } from './kubectl-layer';
 
 export interface KubectlProviderProps {
   /**
@@ -14,8 +14,8 @@ export interface KubectlProviderProps {
 }
 
 export class KubectlProvider extends NestedStack {
-  public static getOrCreate(scope: Construct, cluster: ICluster) {
 
+  public static getOrCreate(scope: Construct, cluster: ICluster) {
     // if this is an "owned" cluster, it has a provider associated with it
     if (cluster instanceof Cluster) {
       return cluster._attachKubectlResourceScope(scope);
@@ -61,13 +61,15 @@ export class KubectlProvider extends NestedStack {
       throw new Error('"kubectlSecurityGroup" is required if "kubectlSubnets" is specified');
     }
 
+    const layer = cluster.kubectlLayer ?? getOrCreateKubectlLayer(this);
+
     const handler = new lambda.Function(this, 'Handler', {
       code: lambda.Code.fromAsset(path.join(__dirname, 'kubectl-handler')),
       runtime: lambda.Runtime.PYTHON_3_7,
       handler: 'index.handler',
       timeout: Duration.minutes(15),
       description: 'onEvent handler for EKS kubectl resource provider',
-      layers: [KubectlLayer.getOrCreate(this, { version: '2.0.0' })],
+      layers: [layer],
       memorySize: 256,
       environment: cluster.kubectlEnvironment,
 
@@ -94,5 +96,21 @@ export class KubectlProvider extends NestedStack {
     this.serviceToken = provider.serviceToken;
     this.roleArn = cluster.kubectlRole.roleArn;
   }
+
 }
 
+/**
+ * Gets or create a singleton instance of KubectlLayer.
+ *
+ * (exported for unit tests).
+ */
+export function getOrCreateKubectlLayer(scope: Construct, props: KubectlLayerProps = {}): KubectlLayer {
+  const stack = Stack.of(scope);
+  const id = 'kubectl-layer-' + (props.version ? props.version : '8C2542BC-BF2B-4DFE-B765-E181FD30A9A0');
+  const exists = stack.node.tryFindChild(id) as KubectlLayer;
+  if (exists) {
+    return exists;
+  }
+
+  return new KubectlLayer(stack, id, props);
+}