docs: Add docs for customAMIs, tests for UserData (#2169)

* docs: Add docs for customAMIs, tests for UserData

examples/provisioner/launchtemplates/al2-custom-ami.yaml

@@ -0,0 +1,40 @@
+# This example provisioner will provision instances using a custom EKS-Optimized AMI that belongs to the
+# AL2 AMIFamily. If your AMIs are built off https://github.com/awslabs/amazon-eks-ami and can be bootstrapped
+# by Karpenter, this may be a good fit for you.
+
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+  name: default
+spec:
+  limits:
+    resources:
+      cpu: 20
+  providerRef:
+    name: al2
+  ttlSecondsAfterEmpty: 30
+---
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: al2
+spec:
+  amiFamily: AL2
+  instanceProfile: myInstanceProfile
+  subnetSelector:
+    karpenter.sh/discovery: my-cluster
+  securityGroupSelector:
+    karpenter.sh/discovery: my-cluster
+  amiSelector:
+    ami-ids: ami-123,ami456
+  userData: |
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+    --BOUNDARY
+    Content-Type: text/x-shellscript; charset="us-ascii"
+
+    #!/bin/bash
+    echo "Running a custom user data script"
+
+    --BOUNDARY--

examples/provisioner/launchtemplates/custom-family.yaml

@@ -0,0 +1,41 @@
+# This example provisioner will provision instances using an AMI that belongs to a custom AMIFamily
+# Keep in mind, that you're in charge of bootstrapping your worker nodes.
+
+apiVersion: karpenter.sh/v1alpha5
+kind: Provisioner
+metadata:
+  name: default
+spec:
+  limits:
+    resources:
+      cpu: 20
+  providerRef:
+    name: custom-family
+  ttlSecondsAfterEmpty: 30
+---
+apiVersion: karpenter.k8s.aws/v1alpha1
+kind: AWSNodeTemplate
+metadata:
+  name: custom-family
+spec:
+  amiFamily: Custom
+  instanceProfile: myInstanceProfile
+  subnetSelector:
+    karpenter.sh/discovery: my-cluster
+  securityGroupSelector:
+    karpenter.sh/discovery: my-cluster
+  amiSelector:
+    ami-ids: ami-123,ami456
+  userData: |
+    MIME-Version: 1.0
+    Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+    --BOUNDARY
+    Content-Type: text/x-shellscript; charset="us-ascii"
+
+    #!/bin/bash
+    echo "Running my custom set-up"
+
+    /etc/eks/bootstrap.sh my-cluster --kubelet-extra-args='--node-labels=foo=bar'
+
+    --BOUNDARY

test/suites/integration/launchtemplates_test.go

@@ -1,7 +1,9 @@
 package integration_test
 
 import (
+	"encoding/base64"
 	"fmt"
+	"io/ioutil"
 	"strconv"
 	"strings"
 
@@ -32,7 +34,7 @@ var _ = Describe("LaunchTemplates", func() {
 				SubnetSelector:        map[string]string{"karpenter.sh/discovery": env.ClusterName},
 				AMIFamily:             &awsv1alpha1.AMIFamilyAL2,
 			},
-			AMISelector:           map[string]string{"aws-ids": customAMI},
+			AMISelector: map[string]string{"aws-ids": customAMI},
 		})
 		provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name}})
 		pod := test.Pod()
@@ -61,6 +63,50 @@ var _ = Describe("LaunchTemplates", func() {
 
 		ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI))))
 	})
+	It("should merge UserData contents for AL2 AMIFamily", func() {
+		content, err := ioutil.ReadFile("testdata/al2_userdata_input.golden")
+		Expect(err).ToNot(HaveOccurred())
+		provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{
+			SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
+			SubnetSelector:        map[string]string{"karpenter.sh/discovery": env.ClusterName},
+			AMIFamily:             &awsv1alpha1.AMIFamilyAL2,
+		},
+			UserData: aws.String(string(content)),
+		})
+		provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name}})
+		pod := test.Pod()
+
+		env.ExpectCreated(pod, provider, provisioner)
+		env.EventuallyExpectHealthy(pod)
+		env.ExpectCreatedNodeCount("==", 1)
+
+		actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value)
+		Expect(err).ToNot(HaveOccurred())
+		// Since the node has joined the cluster, we know our bootstrapping was correct.
+		// Just verify if the UserData contains our custom content too, rather than doing a byte-wise comparison.
+		Expect(string(actualUserData)).To(ContainSubstring("Running custom user data script"))
+	})
+	It("should merge UserData contents for Bottlerocket AMIFamily", func() {
+		content, err := ioutil.ReadFile("testdata/br_userdata_input.golden")
+		Expect(err).ToNot(HaveOccurred())
+		provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{
+			SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName},
+			SubnetSelector:        map[string]string{"karpenter.sh/discovery": env.ClusterName},
+			AMIFamily:             &awsv1alpha1.AMIFamilyBottlerocket,
+		},
+			UserData: aws.String(string(content)),
+		})
+		provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name}})
+		pod := test.Pod()
+
+		env.ExpectCreated(pod, provider, provisioner)
+		env.EventuallyExpectHealthy(pod)
+		env.ExpectCreatedNodeCount("==", 1)
+
+		actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(string(actualUserData)).To(ContainSubstring("kube-api-qps = 30"))
+	})
 })
 
 func ExpectInstance(nodeName string) Assertion {
@@ -77,6 +123,19 @@ func ExpectInstance(nodeName string) Assertion {
 	return Expect(instance.Reservations[0].Instances[0])
 }
 
+func getInstanceAttribute(nodeName string, attribute string) *ec2.DescribeInstanceAttributeOutput {
+	var node v1.Node
+	Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed())
+	providerIDSplit := strings.Split(node.Spec.ProviderID, "/")
+	instanceID := providerIDSplit[len(providerIDSplit)-1]
+	instanceAttribute, err := env.EC2API.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{
+		InstanceId: aws.String(instanceID),
+		Attribute:  aws.String(attribute),
+	})
+	Expect(err).ToNot(HaveOccurred())
+	return instanceAttribute
+}
+
 func selectCustomAMI(amiPath string) string {
 	serverVersion, err := env.KubeClient.Discovery().ServerVersion()
 	Expect(err).To(BeNil())

test/suites/integration/testdata/al2_userdata_input.golden

@@ -0,0 +1,10 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="BOUNDARY"
+
+--BOUNDARY
+Content-Type: text/x-shellscript; charset="us-ascii"
+
+#!/bin/bash
+echo "Running custom user data script"
+
+--BOUNDARY--

test/suites/integration/testdata/br_userdata_input.golden

@@ -0,0 +1,2 @@
+[settings.kubernetes]
+kube-api-qps = 30

website/content/en/preview/AWS/provisioning.md

@@ -54,8 +54,8 @@ You can review these fields [in the code](https://github.com/aws/karpenter/blob{
 ### InstanceProfile
 An `InstanceProfile` is a way to pass a single IAM role to an EC2 instance. Karpenter will not create one automatically.
 A default profile may be specified on the controller, allowing it to be omitted here. If not specified as either a default
-or on the controller, node provisioning will fail. The KarpenterControllerPolicy will also need to have permissions for 
-`iam:PassRole` to the role provided here or provisioning will fail.  
+or on the controller, node provisioning will fail. The KarpenterControllerPolicy will also need to have permissions for
+`iam:PassRole` to the role provided here or provisioning will fail.
 
 ```
 spec:
@@ -210,9 +210,9 @@ spec:
 
 ### Amazon Machine Image (AMI) Family
 
-The AMI used when provisioning nodes can be controlled by the `amiFamily` field. Based on the value set for `amiFamily`, Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM).
+The AMI used when provisioning nodes can be controlled by the `amiFamily` field. Based on the value set for `amiFamily`, Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). When an `amiFamily` of `Custom` is chosen, then an `amiSelector` must be specified that informs Karpenter on which custom AMIs are to be used.
 
-Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, and `Ubuntu`. GPUs are only supported with `AL2` and `Bottlerocket`.
+Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu` and `Custom`. GPUs are only supported with `AL2` and `Bottlerocket`.
 
 Note: If a custom launch template is specified, then the AMI value in the launch template is used rather than the `amiFamily` value.
 
@@ -252,6 +252,46 @@ spec:
 You can control the UserData that needs to be applied to your worker nodes via this field. Review the [Custom UserData documentation](../user-data/) to learn the necessary steps
 If you need to specify a launch template in addition to UserData, then review the [Launch Template documentation](../launch-templates/) instead and utilize the `spec.providerRef.launchTemplate` field.
 
+### AMISelector
+
+AMISelector is used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html), similar to `subnetSelector`. This field is optional, and Karpenter will use the latest EKS-optimized AMIs if an amiSelector is not specified.
+
+EC2 AMIs may be specified by any AWS tag, including `Name`. Selecting tag values using wildcards (`*`) is supported.
+
+EC2 AMI IDs may be specified by using the key `aws-ids` and then passing the IDs as a comma-separated string value.
+
+* When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements.
+* If multiple AMIs are found that can be used, Karpenter will randomly choose any one.
+* If no AMIs are found that can be used, then no nodes will be provisioned.
+
+For additional data on how UserData is configured for Custom AMIs, and how more requirements can be specified for custom AMIs, follow [this documentation](../user-data/#custom-amis).
+
+**Examples**
+
+Select all AMIs with a specified tag:
+```
+  amiSelector:
+    karpenter.sh/discovery/MyClusterName: '*'
+```
+
+Select AMIs by name:
+```
+  amiSelector:
+    Name: my-ami
+```
+
+Select AMIs by an arbitrary AWS tag key/value pair:
+```
+  amiSelector:
+    MySubnetTag: value
+```
+
+Specify AMIs explicitly by ID:
+```yaml
+  amiSelector:
+    aws-ids: "ami-123,ami-456"
+```
+
 ## spec.provider (Deprecated)
 
 Prior to the introduction of `spec.providerRef`, parameters for the AWS Cloud Provider could be specified within the Provisioner itself through the `spec.provider` field. This field in the Provisioners has now been deprecated, and all fields previously specified through the ProvisionerSpec can now be specified in the `AWSNodeTemplate` CRD instead. See the [upgrade guide for more information](../upgrade-guide/_index.md). New parameters can only be specified in the `AWSNodeTemplate` CRD.

website/content/en/preview/AWS/user-data.md

@@ -1,20 +1,20 @@
 ---
-title: "User Data Configuration"
-linkTitle: "UserData"
+title: "Custom User Data and AMI Configuration"
+linkTitle: "Custom User Data and AMI"
 weight: 10
 description: >
-  Learn how to configure custom UserData with Karpenter
+  Learn how to configure custom UserData and AMIs with Karpenter
 ---
 
-This document describes how you can customize the UserData that will be specified on your EC2 worker nodes, without using a launch template.
+This document describes how you can customize the UserData and AMIs for your EC2 worker nodes, without using a launch template.
 
 ## Configuration
 
-In order to specify custom user data, you must include it within a AWSNodeTemplate resource. You can then reference this AWSNodeTemplate resource through `spec.providerRef` in your provisioner.
+In order to specify custom user data and AMIs, you must include them within a AWSNodeTemplate resource. You can then reference this AWSNodeTemplate resource through `spec.providerRef` in your provisioner.
 
 **Examples**
 
-Your UserData can be added to `spec.userData` in the `AWSNodeTemplate` resource like this -
+Your UserData and AMIs can be added to `spec.userData` and `spec.amiSelector` respectively in the `AWSNodeTemplate` resource -
 ```yaml
 apiVersion: karpenter.k8s.aws/v1alpha1
 kind: AWSNodeTemplate
@@ -32,9 +32,11 @@ spec:
     kube-api-qps = 30
     [settings.kubernetes.eviction-hard]
     "memory.available" = "20%"
+  amiSelector:
+    karpenter.sh/discovery: my-cluster
 ```
 
-For more examples on configuring UserData, see the examples for [AL2](https://github.com/aws/karpenter/blob/main/examples/provisioner/al2-custom-userdata.yaml) and [Bottlerocket](https://github.com/aws/karpenter/blob/main/examples/provisioner/br-custom-userdata.yaml).
+For more examples on configuring these fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/provisioner/launchtemplates).
 
 ## UserData Content and Merge Semantics
 
@@ -123,3 +125,50 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 --kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test  --max-pods=110'
 --//--
 ```
+
+
+## Custom AMIs
+
+You can specify a set of AMIs for a provisioner to use by specifying an AMISelector that identifies AMIs to use through EC2 tags or via a comma-separated list.
+
+### Defining AMI constraints
+
+Karpenter will automatically determine the architecture that an EC2 AMI is compatible with (amd64, arm64), but other constraints of an AMI can be expressed as tags on the EC2 AMI.
+For example, if you want to limit an EC2 AMI to only be used with instanceTypes that have an `nvidia` GPU, you can specify an EC2 tag with a key of `karpenter.k8s.aws/instance-gpu-manufacturer` and value `nvidia` on that AMI.
+
+All labels defined [in the scheduling documentation](../../tasks/scheduling#supported-labels) can be used as requirements for an EC2 AMI.
+
+```
+> aws ec2 describe-images --image-id ami-123 --query Images[0].Tags
+[
+    {
+        "Key": "karpenter.sh/discovery",
+        "Value": "my-cluster"
+    },
+    {
+        "Key": "Name",
+        "Value": "amazon-eks-node-1.21-customized-v0"
+    },
+    {
+        "Key": "karpenter.k8s.aws/instance-gpu-manufacturer",
+        "Value": "nvidia"
+    }
+]
+```
+
+
+###  AMIFamily
+
+When you give Karpenter an AMI ID to use, you can specify which AMIFamily they belong to. This will determine how Karpenter should use your AMI.
+For example, if you define the `AMIFamily` to be `AL2`, then Karpenter will assume that a worker node using that AMI should be bootstrapped in the same manner as EKS-optimized AL2 AMIs. This is useful when your custom images are variants of EKS-optimized AMIs and there are no differences in how bootstrapping needs to be performed.
+
+When the `AMIFamily` is set to `Custom`, then Karpenter will not attempt to bootstrap the worker node. You must set the necessary commands through `spec.UserData` to ensure that your worker node joins the cluster.
+
+
+### Binpacking semantics for AMIFamily
+
+In order for Karpenter to accurately binpack your pods in a worker node, it needs to know the eventual allocatable capacity on your node. This capacity has several dimensions (cpu, memory, ephemeral-storage) and is a function of the instanceType as well as the AMI.
+
+* When the AMIFamily is *`AL2`, `Bottlerocket` or `Ubuntu`*, Karpenter will bin-pack your pods in the same way as other EKS-optimized AMIs of that family.
+* When the AMIFamily is *`Custom`*, Karpenter assumes that the amount of allocatable cpu, memory and ephemeral-storage is identical to `AL2` EKS-Optimized AMIs, regardless of how the node is being bootstrapped.
+  * When the AMIFamily is *`Custom`*, Karpenter has no way of knowing which ephemeral volume will be used for pods. Therefore, it will default to using the last volume in `spec.blockDeviceMappings` to determine the total available ephemeral capacity on a worker node.

website/content/en/preview/upgrade-guide/_index.md

@@ -114,6 +114,7 @@ aws ec2 delete-launch-template --launch-template-id <LAUNCH_TEMPLATE_ID>
     operator: Exists
 ```
 
+* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here](../aws/provisioning/#amiselector) to get started
 * v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overriden by using `--set tolerations[0]=null`.
 
 ## Upgrading to v0.13.0+