From 1d3159c53b0776192d46b732e114407ae70e8111 Mon Sep 17 00:00:00 2001 From: Suket Sharma Date: Tue, 26 Jul 2022 09:43:58 -0700 Subject: [PATCH] docs: Add docs for customAMIs, tests for UserData (#2169) * docs: Add docs for customAMIs, tests for UserData --- .../launchtemplates/al2-custom-ami.yaml | 40 ++++++++++++ .../al2-custom-userdata.yaml | 0 .../br-custom-userdata.yaml | 0 .../launchtemplates/custom-family.yaml | 41 ++++++++++++ .../integration/launchtemplates_test.go | 61 +++++++++++++++++- .../testdata/al2_userdata_input.golden | 10 +++ .../testdata/br_userdata_input.golden | 2 + .../content/en/preview/AWS/provisioning.md | 48 ++++++++++++-- website/content/en/preview/AWS/user-data.md | 63 ++++++++++++++++--- .../en/preview/upgrade-guide/_index.md | 1 + 10 files changed, 254 insertions(+), 12 deletions(-) create mode 100644 examples/provisioner/launchtemplates/al2-custom-ami.yaml rename examples/provisioner/{ => launchtemplates}/al2-custom-userdata.yaml (100%) rename examples/provisioner/{ => launchtemplates}/br-custom-userdata.yaml (100%) create mode 100644 examples/provisioner/launchtemplates/custom-family.yaml create mode 100644 test/suites/integration/testdata/al2_userdata_input.golden create mode 100644 test/suites/integration/testdata/br_userdata_input.golden diff --git a/examples/provisioner/launchtemplates/al2-custom-ami.yaml b/examples/provisioner/launchtemplates/al2-custom-ami.yaml new file mode 100644 index 000000000000..f4f6019f59d1 --- /dev/null +++ b/examples/provisioner/launchtemplates/al2-custom-ami.yaml @@ -0,0 +1,40 @@ +# This example provisioner will provision instances using a custom EKS-Optimized AMI that belongs to the +# AL2 AMIFamily. If your AMIs are built off https://github.com/awslabs/amazon-eks-ami and can be bootstrapped +# by Karpenter, this may be a good fit for you. + +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + limits: + resources: + cpu: 20 + providerRef: + name: al2 + ttlSecondsAfterEmpty: 30 +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: al2 +spec: + amiFamily: AL2 + instanceProfile: myInstanceProfile + subnetSelector: + karpenter.sh/discovery: my-cluster + securityGroupSelector: + karpenter.sh/discovery: my-cluster + amiSelector: + ami-ids: ami-123,ami456 + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + echo "Running a custom user data script" + + --BOUNDARY-- diff --git a/examples/provisioner/al2-custom-userdata.yaml b/examples/provisioner/launchtemplates/al2-custom-userdata.yaml similarity index 100% rename from examples/provisioner/al2-custom-userdata.yaml rename to examples/provisioner/launchtemplates/al2-custom-userdata.yaml diff --git a/examples/provisioner/br-custom-userdata.yaml b/examples/provisioner/launchtemplates/br-custom-userdata.yaml similarity index 100% rename from examples/provisioner/br-custom-userdata.yaml rename to examples/provisioner/launchtemplates/br-custom-userdata.yaml diff --git a/examples/provisioner/launchtemplates/custom-family.yaml b/examples/provisioner/launchtemplates/custom-family.yaml new file mode 100644 index 000000000000..ccebee2e29f3 --- /dev/null +++ b/examples/provisioner/launchtemplates/custom-family.yaml @@ -0,0 +1,41 @@ +# This example provisioner will provision instances using an AMI that belongs to a custom AMIFamily +# Keep in mind, that you're in charge of bootstrapping your worker nodes. + +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + limits: + resources: + cpu: 20 + providerRef: + name: custom-family + ttlSecondsAfterEmpty: 30 +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: custom-family +spec: + amiFamily: Custom + instanceProfile: myInstanceProfile + subnetSelector: + karpenter.sh/discovery: my-cluster + securityGroupSelector: + karpenter.sh/discovery: my-cluster + amiSelector: + ami-ids: ami-123,ami456 + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + echo "Running my custom set-up" + + /etc/eks/bootstrap.sh my-cluster --kubelet-extra-args='--node-labels=foo=bar' + + --BOUNDARY diff --git a/test/suites/integration/launchtemplates_test.go b/test/suites/integration/launchtemplates_test.go index d7e64ec1d687..f1dbf73a5822 100644 --- a/test/suites/integration/launchtemplates_test.go +++ b/test/suites/integration/launchtemplates_test.go @@ -1,7 +1,9 @@ package integration_test import ( + "encoding/base64" "fmt" + "io/ioutil" "strconv" "strings" @@ -32,7 +34,7 @@ var _ = Describe("LaunchTemplates", func() { SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &awsv1alpha1.AMIFamilyAL2, }, - AMISelector: map[string]string{"aws-ids": customAMI}, + AMISelector: map[string]string{"aws-ids": customAMI}, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name}}) pod := test.Pod() @@ -61,6 +63,50 @@ var _ = Describe("LaunchTemplates", func() { ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) }) + It("should merge UserData contents for AL2 AMIFamily", func() { + content, err := ioutil.ReadFile("testdata/al2_userdata_input.golden") + Expect(err).ToNot(HaveOccurred()) + provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + AMIFamily: &awsv1alpha1.AMIFamilyAL2, + }, + UserData: aws.String(string(content)), + }) + provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name}}) + pod := test.Pod() + + env.ExpectCreated(pod, provider, provisioner) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) + Expect(err).ToNot(HaveOccurred()) + // Since the node has joined the cluster, we know our bootstrapping was correct. + // Just verify if the UserData contains our custom content too, rather than doing a byte-wise comparison. + Expect(string(actualUserData)).To(ContainSubstring("Running custom user data script")) + }) + It("should merge UserData contents for Bottlerocket AMIFamily", func() { + content, err := ioutil.ReadFile("testdata/br_userdata_input.golden") + Expect(err).ToNot(HaveOccurred()) + provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: awsv1alpha1.AWS{ + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + AMIFamily: &awsv1alpha1.AMIFamilyBottlerocket, + }, + UserData: aws.String(string(content)), + }) + provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.ProviderRef{Name: provider.Name}}) + pod := test.Pod() + + env.ExpectCreated(pod, provider, provisioner) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + actualUserData, err := base64.StdEncoding.DecodeString(*getInstanceAttribute(pod.Spec.NodeName, "userData").UserData.Value) + Expect(err).ToNot(HaveOccurred()) + Expect(string(actualUserData)).To(ContainSubstring("kube-api-qps = 30")) + }) }) func ExpectInstance(nodeName string) Assertion { @@ -77,6 +123,19 @@ func ExpectInstance(nodeName string) Assertion { return Expect(instance.Reservations[0].Instances[0]) } +func getInstanceAttribute(nodeName string, attribute string) *ec2.DescribeInstanceAttributeOutput { + var node v1.Node + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) + providerIDSplit := strings.Split(node.Spec.ProviderID, "/") + instanceID := providerIDSplit[len(providerIDSplit)-1] + instanceAttribute, err := env.EC2API.DescribeInstanceAttribute(&ec2.DescribeInstanceAttributeInput{ + InstanceId: aws.String(instanceID), + Attribute: aws.String(attribute), + }) + Expect(err).ToNot(HaveOccurred()) + return instanceAttribute +} + func selectCustomAMI(amiPath string) string { serverVersion, err := env.KubeClient.Discovery().ServerVersion() Expect(err).To(BeNil()) diff --git a/test/suites/integration/testdata/al2_userdata_input.golden b/test/suites/integration/testdata/al2_userdata_input.golden new file mode 100644 index 000000000000..afc1580817ae --- /dev/null +++ b/test/suites/integration/testdata/al2_userdata_input.golden @@ -0,0 +1,10 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script" + +--BOUNDARY-- diff --git a/test/suites/integration/testdata/br_userdata_input.golden b/test/suites/integration/testdata/br_userdata_input.golden new file mode 100644 index 000000000000..47eb46dc26f1 --- /dev/null +++ b/test/suites/integration/testdata/br_userdata_input.golden @@ -0,0 +1,2 @@ +[settings.kubernetes] +kube-api-qps = 30 diff --git a/website/content/en/preview/AWS/provisioning.md b/website/content/en/preview/AWS/provisioning.md index a5e8b094bcbd..e5485c32cb12 100644 --- a/website/content/en/preview/AWS/provisioning.md +++ b/website/content/en/preview/AWS/provisioning.md @@ -54,8 +54,8 @@ You can review these fields [in the code](https://github.com/aws/karpenter/blob{ ### InstanceProfile An `InstanceProfile` is a way to pass a single IAM role to an EC2 instance. Karpenter will not create one automatically. A default profile may be specified on the controller, allowing it to be omitted here. If not specified as either a default -or on the controller, node provisioning will fail. The KarpenterControllerPolicy will also need to have permissions for -`iam:PassRole` to the role provided here or provisioning will fail. +or on the controller, node provisioning will fail. The KarpenterControllerPolicy will also need to have permissions for +`iam:PassRole` to the role provided here or provisioning will fail. ``` spec: @@ -210,9 +210,9 @@ spec: ### Amazon Machine Image (AMI) Family -The AMI used when provisioning nodes can be controlled by the `amiFamily` field. Based on the value set for `amiFamily`, Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). +The AMI used when provisioning nodes can be controlled by the `amiFamily` field. Based on the value set for `amiFamily`, Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). When an `amiFamily` of `Custom` is chosen, then an `amiSelector` must be specified that informs Karpenter on which custom AMIs are to be used. -Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, and `Ubuntu`. GPUs are only supported with `AL2` and `Bottlerocket`. +Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu` and `Custom`. GPUs are only supported with `AL2` and `Bottlerocket`. Note: If a custom launch template is specified, then the AMI value in the launch template is used rather than the `amiFamily` value. @@ -252,6 +252,46 @@ spec: You can control the UserData that needs to be applied to your worker nodes via this field. Review the [Custom UserData documentation](../user-data/) to learn the necessary steps If you need to specify a launch template in addition to UserData, then review the [Launch Template documentation](../launch-templates/) instead and utilize the `spec.providerRef.launchTemplate` field. +### AMISelector + +AMISelector is used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html), similar to `subnetSelector`. This field is optional, and Karpenter will use the latest EKS-optimized AMIs if an amiSelector is not specified. + +EC2 AMIs may be specified by any AWS tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. + +EC2 AMI IDs may be specified by using the key `aws-ids` and then passing the IDs as a comma-separated string value. + +* When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. +* If multiple AMIs are found that can be used, Karpenter will randomly choose any one. +* If no AMIs are found that can be used, then no nodes will be provisioned. + +For additional data on how UserData is configured for Custom AMIs, and how more requirements can be specified for custom AMIs, follow [this documentation](../user-data/#custom-amis). + +**Examples** + +Select all AMIs with a specified tag: +``` + amiSelector: + karpenter.sh/discovery/MyClusterName: '*' +``` + +Select AMIs by name: +``` + amiSelector: + Name: my-ami +``` + +Select AMIs by an arbitrary AWS tag key/value pair: +``` + amiSelector: + MySubnetTag: value +``` + +Specify AMIs explicitly by ID: +```yaml + amiSelector: + aws-ids: "ami-123,ami-456" +``` + ## spec.provider (Deprecated) Prior to the introduction of `spec.providerRef`, parameters for the AWS Cloud Provider could be specified within the Provisioner itself through the `spec.provider` field. This field in the Provisioners has now been deprecated, and all fields previously specified through the ProvisionerSpec can now be specified in the `AWSNodeTemplate` CRD instead. See the [upgrade guide for more information](../upgrade-guide/_index.md). New parameters can only be specified in the `AWSNodeTemplate` CRD. diff --git a/website/content/en/preview/AWS/user-data.md b/website/content/en/preview/AWS/user-data.md index a9aa2a3ed24b..8f0b5b52aaec 100644 --- a/website/content/en/preview/AWS/user-data.md +++ b/website/content/en/preview/AWS/user-data.md @@ -1,20 +1,20 @@ --- -title: "User Data Configuration" -linkTitle: "UserData" +title: "Custom User Data and AMI Configuration" +linkTitle: "Custom User Data and AMI" weight: 10 description: > - Learn how to configure custom UserData with Karpenter + Learn how to configure custom UserData and AMIs with Karpenter --- -This document describes how you can customize the UserData that will be specified on your EC2 worker nodes, without using a launch template. +This document describes how you can customize the UserData and AMIs for your EC2 worker nodes, without using a launch template. ## Configuration -In order to specify custom user data, you must include it within a AWSNodeTemplate resource. You can then reference this AWSNodeTemplate resource through `spec.providerRef` in your provisioner. +In order to specify custom user data and AMIs, you must include them within a AWSNodeTemplate resource. You can then reference this AWSNodeTemplate resource through `spec.providerRef` in your provisioner. **Examples** -Your UserData can be added to `spec.userData` in the `AWSNodeTemplate` resource like this - +Your UserData and AMIs can be added to `spec.userData` and `spec.amiSelector` respectively in the `AWSNodeTemplate` resource - ```yaml apiVersion: karpenter.k8s.aws/v1alpha1 kind: AWSNodeTemplate @@ -32,9 +32,11 @@ spec: kube-api-qps = 30 [settings.kubernetes.eviction-hard] "memory.available" = "20%" + amiSelector: + karpenter.sh/discovery: my-cluster ``` -For more examples on configuring UserData, see the examples for [AL2](https://github.com/aws/karpenter/blob/main/examples/provisioner/al2-custom-userdata.yaml) and [Bottlerocket](https://github.com/aws/karpenter/blob/main/examples/provisioner/br-custom-userdata.yaml). +For more examples on configuring these fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/provisioner/launchtemplates). ## UserData Content and Merge Semantics @@ -123,3 +125,50 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 --kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test --max-pods=110' --//-- ``` + + +## Custom AMIs + +You can specify a set of AMIs for a provisioner to use by specifying an AMISelector that identifies AMIs to use through EC2 tags or via a comma-separated list. + +### Defining AMI constraints + +Karpenter will automatically determine the architecture that an EC2 AMI is compatible with (amd64, arm64), but other constraints of an AMI can be expressed as tags on the EC2 AMI. +For example, if you want to limit an EC2 AMI to only be used with instanceTypes that have an `nvidia` GPU, you can specify an EC2 tag with a key of `karpenter.k8s.aws/instance-gpu-manufacturer` and value `nvidia` on that AMI. + +All labels defined [in the scheduling documentation](../../tasks/scheduling#supported-labels) can be used as requirements for an EC2 AMI. + +``` +> aws ec2 describe-images --image-id ami-123 --query Images[0].Tags +[ + { + "Key": "karpenter.sh/discovery", + "Value": "my-cluster" + }, + { + "Key": "Name", + "Value": "amazon-eks-node-1.21-customized-v0" + }, + { + "Key": "karpenter.k8s.aws/instance-gpu-manufacturer", + "Value": "nvidia" + } +] +``` + + +### AMIFamily + +When you give Karpenter an AMI ID to use, you can specify which AMIFamily they belong to. This will determine how Karpenter should use your AMI. +For example, if you define the `AMIFamily` to be `AL2`, then Karpenter will assume that a worker node using that AMI should be bootstrapped in the same manner as EKS-optimized AL2 AMIs. This is useful when your custom images are variants of EKS-optimized AMIs and there are no differences in how bootstrapping needs to be performed. + +When the `AMIFamily` is set to `Custom`, then Karpenter will not attempt to bootstrap the worker node. You must set the necessary commands through `spec.UserData` to ensure that your worker node joins the cluster. + + +### Binpacking semantics for AMIFamily + +In order for Karpenter to accurately binpack your pods in a worker node, it needs to know the eventual allocatable capacity on your node. This capacity has several dimensions (cpu, memory, ephemeral-storage) and is a function of the instanceType as well as the AMI. + +* When the AMIFamily is *`AL2`, `Bottlerocket` or `Ubuntu`*, Karpenter will bin-pack your pods in the same way as other EKS-optimized AMIs of that family. +* When the AMIFamily is *`Custom`*, Karpenter assumes that the amount of allocatable cpu, memory and ephemeral-storage is identical to `AL2` EKS-Optimized AMIs, regardless of how the node is being bootstrapped. + * When the AMIFamily is *`Custom`*, Karpenter has no way of knowing which ephemeral volume will be used for pods. Therefore, it will default to using the last volume in `spec.blockDeviceMappings` to determine the total available ephemeral capacity on a worker node. \ No newline at end of file diff --git a/website/content/en/preview/upgrade-guide/_index.md b/website/content/en/preview/upgrade-guide/_index.md index 3b71b9397bcb..618f8b11b9d8 100644 --- a/website/content/en/preview/upgrade-guide/_index.md +++ b/website/content/en/preview/upgrade-guide/_index.md @@ -114,6 +114,7 @@ aws ec2 delete-launch-template --launch-template-id operator: Exists ``` +* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here](../aws/provisioning/#amiselector) to get started * v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overriden by using `--set tolerations[0]=null`. ## Upgrading to v0.13.0+