diff --git a/charts/index.yaml b/charts/index.yaml index 3b34b489a731..d5c342124512 100644 --- a/charts/index.yaml +++ b/charts/index.yaml @@ -1,9 +1,19 @@ apiVersion: v1 entries: karpenter: + - apiVersion: v2 + appVersion: 0.5.5 + created: "2022-01-19T10:29:50.886257-08:00" + description: A Helm chart for https://github.com/aws/karpenter/. + digest: 2bd507dd53f10c82bdfe049f0bb4c0193ecf684d73b44ff1bbb5cf2f16f0fa25 + name: karpenter + type: application + urls: + - karpenter-0.5.5.tgz + version: 0.5.5 - apiVersion: v2 appVersion: 0.5.4 - created: "2022-01-07T19:18:47.586454-05:00" + created: "2022-01-19T10:29:50.885335-08:00" description: A Helm chart for https://github.com/aws/karpenter/. digest: 0c613878bc1827f2188a78b5d2a2945c417c29c699f8f9c5fa7f4e2cbdeefebb name: karpenter @@ -12,7 +22,7 @@ entries: - karpenter-0.5.4.tgz version: 0.5.4 - apiVersion: v2 - created: "2022-01-07T19:18:47.585681-05:00" + created: "2022-01-19T10:29:50.883973-08:00" description: A Helm chart for https://github.com/aws/karpenter/. digest: f6bb831f6ceea8b90634f9a6841e77de97ad1e3350437cb66cc50f14b7c3becc name: karpenter @@ -21,7 +31,7 @@ entries: - karpenter-0.5.3.tgz version: 0.5.3 - apiVersion: v2 - created: "2022-01-07T19:18:47.584568-05:00" + created: "2022-01-19T10:29:50.883129-08:00" description: A Helm chart for https://github.com/aws/karpenter/. digest: 45f6262a018c11c0c3068e29dc2e1cf337cba8c390ec269b9b36c6d8bdac581e name: karpenter @@ -30,7 +40,7 @@ entries: - karpenter-0.5.2.tgz version: 0.5.2 - apiVersion: v2 - created: "2022-01-07T19:18:47.583373-05:00" + created: "2022-01-19T10:29:50.88206-08:00" description: A Helm chart for https://github.com/aws/karpenter/. digest: 4247c15ec571f82025c80ceaa9354ce3ee3b209e4ba15a0ca3c36a1f90a859ec name: karpenter @@ -39,7 +49,7 @@ entries: - karpenter-0.5.1.tgz version: 0.5.1 - apiVersion: v2 - created: "2022-01-07T19:18:47.581513-05:00" + created: "2022-01-19T10:29:50.880863-08:00" description: A Helm chart for https://github.com/aws/karpenter/. digest: 6d49a00dca8a6f8d7938e9508228af085d85a2746e2f742c895b954fe71292df name: karpenter @@ -48,7 +58,7 @@ entries: - karpenter-0.5.0.tgz version: 0.5.0 - apiVersion: v2 - created: "2022-01-07T19:18:47.580501-05:00" + created: "2022-01-19T10:29:50.879852-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 70c5bbea1016aa56da5227b7f5d7cf5700c4d38ff1814f8e28072c77d8a02c20 name: karpenter @@ -57,7 +67,7 @@ entries: - karpenter-0.4.3.tgz version: 0.4.3 - apiVersion: v2 - created: "2022-01-07T19:18:47.579615-05:00" + created: "2022-01-19T10:29:50.87867-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 558f26b8786384766407c5f86cff5180066c4fa68fe9e3cf7b782ec8564f04ff name: karpenter @@ -66,7 +76,7 @@ entries: - karpenter-0.4.2.tgz version: 0.4.2 - apiVersion: v2 - created: "2022-01-07T19:18:47.578645-05:00" + created: "2022-01-19T10:29:50.877727-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: ec4bcf2a7f279ad97b8749a7f1a8edd527a09f8c00614d911a773843bd44a58b name: karpenter @@ -75,7 +85,7 @@ entries: - karpenter-0.4.1.tgz version: 0.4.1 - apiVersion: v2 - created: "2022-01-07T19:18:47.577617-05:00" + created: "2022-01-19T10:29:50.876645-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: d4ea011979906ad22b2345d0eb4c941445e89c1b0561c6ae44bb71fdb641c161 name: karpenter @@ -84,7 +94,7 @@ entries: - karpenter-0.4.0.tgz version: 0.4.0 - apiVersion: v2 - created: "2022-01-07T19:18:47.576685-05:00" + created: "2022-01-19T10:29:50.875304-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: e8b9b3dd447d0ab48b66156bf88ae924121e5780d741ee18cc9a43a90a6f4290 name: karpenter @@ -93,7 +103,7 @@ entries: - karpenter-0.3.4.tgz version: 0.3.4 - apiVersion: v2 - created: "2022-01-07T19:18:47.575849-05:00" + created: "2022-01-19T10:29:50.874379-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 0d8bc4c74416aeb5233a0a65c4b4fd678b7c961f1dca10604e33337715b585ff name: karpenter @@ -102,7 +112,7 @@ entries: - karpenter-0.3.3.tgz version: 0.3.3 - apiVersion: v2 - created: "2022-01-07T19:18:47.574718-05:00" + created: "2022-01-19T10:29:50.873408-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: e24d1ca8364e92d161f8295b552ec101a72491c62926dd67779d4e4a7e6b5756 name: karpenter @@ -111,7 +121,7 @@ entries: - karpenter-0.3.2.tgz version: 0.3.2 - apiVersion: v2 - created: "2022-01-07T19:18:47.573945-05:00" + created: "2022-01-19T10:29:50.872612-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 86c9a22a01247b2ed564baa12b88ed2df6556c9e8cb6aca9b990212c1e809c45 name: karpenter @@ -120,7 +130,7 @@ entries: - karpenter-0.3.1.tgz version: 0.3.1 - apiVersion: v2 - created: "2022-01-07T19:18:47.573196-05:00" + created: "2022-01-19T10:29:50.871379-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 6d86156704c2c7eac8094dbe57458a3b1334eb38626b6da4ccd60c5cd65e0250 name: karpenter @@ -129,7 +139,7 @@ entries: - karpenter-0.3.0.tgz version: v0.3.0 - apiVersion: v2 - created: "2022-01-07T19:18:47.572325-05:00" + created: "2022-01-19T10:29:50.870614-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 5b19f5da4dd3d3e147f3e93bf168d1cd24106e12eb7fe44ad88850f73ceb82f0 name: karpenter @@ -138,7 +148,7 @@ entries: - karpenter-0.2.9.tgz version: v0.2.9 - apiVersion: v2 - created: "2022-01-07T19:18:47.571476-05:00" + created: "2022-01-19T10:29:50.869783-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 8b704acc1a9c018da774f62c6be824e7389bb482437d318902d01237e3cacfc5 name: karpenter @@ -147,7 +157,7 @@ entries: - karpenter-0.2.8.tgz version: v0.2.8 - apiVersion: v2 - created: "2022-01-07T19:18:47.570663-05:00" + created: "2022-01-19T10:29:50.869-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 048ec496d46d8ab534bc8af6090699fedd2aaf45c8e1658636c7a44afdb6e898 name: karpenter @@ -156,7 +166,7 @@ entries: - karpenter-0.2.7.tgz version: v0.2.7 - apiVersion: v2 - created: "2022-01-07T19:18:47.568735-05:00" + created: "2022-01-19T10:29:50.868216-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 48008ba427baa5206bd59abe9e25005b0b6673f03e705f2149717bb89156873d name: karpenter @@ -165,7 +175,7 @@ entries: - karpenter-0.2.6.tgz version: v0.2.6 - apiVersion: v2 - created: "2022-01-07T19:18:47.567902-05:00" + created: "2022-01-19T10:29:50.867483-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 208d0c14d2cdbc8c387dc92a473b625f905e829486edd5fd007eb56c9f896682 name: karpenter @@ -174,7 +184,7 @@ entries: - karpenter-0.2.5.tgz version: v0.2.5 - apiVersion: v2 - created: "2022-01-07T19:18:47.56709-05:00" + created: "2022-01-19T10:29:50.866684-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: ed7d1d08bde38f41a6bc8a4bf93b4dd85a3b6e5f526e44324483eadc9faeea6d name: karpenter @@ -183,7 +193,7 @@ entries: - karpenter-0.2.4.tgz version: v0.2.4 - apiVersion: v2 - created: "2022-01-07T19:18:47.566239-05:00" + created: "2022-01-19T10:29:50.865841-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 902bd53be060893d4bfe3c0f57ae831448aa8790930d153666429ea5472d824b name: karpenter @@ -192,7 +202,7 @@ entries: - karpenter-0.2.3.tgz version: v0.2.3 - apiVersion: v2 - created: "2022-01-07T19:18:47.56563-05:00" + created: "2022-01-19T10:29:50.865333-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 90d075cbc08871ffb56b2e530fdf304b6af32c76670fcdd299af87d3810d4651 name: karpenter @@ -201,7 +211,7 @@ entries: - karpenter-0.2.2.tgz version: v0.2.2 - apiVersion: v2 - created: "2022-01-07T19:18:47.564895-05:00" + created: "2022-01-19T10:29:50.864703-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 87e161d40c65dc58a3963f17d70cc165a5bf46155f723e487486f57d209e50d1 name: karpenter @@ -210,7 +220,7 @@ entries: - karpenter-0.2.1.tgz version: v0.2.1 - apiVersion: v2 - created: "2022-01-07T19:18:47.564221-05:00" + created: "2022-01-19T10:29:50.864135-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 552bdc17f5625e4696bb7419284026f4291428877092ec5d984f486a2b812d6f name: karpenter @@ -219,7 +229,7 @@ entries: - karpenter-0.2.0.tgz version: v0.2.0 - apiVersion: v2 - created: "2022-01-07T19:18:47.563516-05:00" + created: "2022-01-19T10:29:50.863558-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 1a597c415201e61576b83ee6ec3e24b99281805b3be8141b0a344c6f014d4e15 name: karpenter @@ -228,7 +238,7 @@ entries: - karpenter-0.1.3.tgz version: v0.1.3 - apiVersion: v2 - created: "2022-01-07T19:18:47.562265-05:00" + created: "2022-01-19T10:29:50.862911-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 6a6753731aef19db2aae72b6bfc1535917053d87f706579e158cae98c23887b4 name: karpenter @@ -237,7 +247,7 @@ entries: - karpenter-0.1.2.tgz version: v0.1.2 - apiVersion: v2 - created: "2022-01-07T19:18:47.561423-05:00" + created: "2022-01-19T10:29:50.862102-08:00" description: A Helm chart for https://github.com/awslabs/karpenter/. digest: 39685c8cbe9a757ca48721aed08b49111fef18bc2a9f67d3223f19d0706f09f7 name: karpenter @@ -245,4 +255,4 @@ entries: urls: - karpenter-0.1.1.tgz version: v0.1.1 -generated: "2022-01-07T19:18:47.560039-05:00" +generated: "2022-01-19T10:29:50.860718-08:00" diff --git a/charts/karpenter-0.5.5.tgz b/charts/karpenter-0.5.5.tgz new file mode 100644 index 000000000000..220943539165 Binary files /dev/null and b/charts/karpenter-0.5.5.tgz differ diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index a258747e4a57..13647e70bca9 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 -appVersion: 0.5.4 +appVersion: 0.5.5 name: karpenter description: A Helm chart for https://github.com/aws/karpenter/. type: application -version: 0.5.4 +version: 0.5.5 diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md index b3c410dd8041..219b03ca8d02 100644 --- a/charts/karpenter/README.md +++ b/charts/karpenter/README.md @@ -2,7 +2,7 @@ A Helm chart for https://github.com/aws/karpenter/. -![Version: 0.5.4](https://img.shields.io/badge/Version-0.5.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.5.4](https://img.shields.io/badge/AppVersion-0.5.4-informational?style=flat-square) +![Version: 0.5.5](https://img.shields.io/badge/Version-0.5.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.5.5](https://img.shields.io/badge/AppVersion-0.5.5-informational?style=flat-square) ## Installing the Chart @@ -12,7 +12,7 @@ To install the chart with the release name `karpenter`: $ helm repo add karpenter https://charts.karpenter.sh $ helm repo update $ helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ - --create-namespace --set serviceAccount.create=false --version 0.5.4 \ + --create-namespace --set serviceAccount.create=false --version 0.5.5 \ --set controller.clusterName=${CLUSTER_NAME} \ --set controller.clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ --wait # for the defaulting webhook to install before creating a Provisioner @@ -29,7 +29,7 @@ You can follow the detailed installation instruction [here](https://karpenter.sh | controller.clusterEndpoint | string | `""` | Cluster endpoint | | controller.clusterName | string | `""` | Cluster name | | controller.env | list | `[]` | Additional environment variables to run with | -| controller.image | string | `"public.ecr.aws/karpenter/controller:v0.5.4@sha256:19ebf83d64fa41d75fb19ae5047f54b1c66423b4ab5ceef36ae99a0daaad1895"` | Image to use for the Karpenter controller | +| controller.image | string | `"public.ecr.aws/karpenter/controller:v0.5.5@sha256:f955fbf012f200048f022f361d48dd83e9ef83d61faa88086f0377cade0492ad"` | Image to use for the Karpenter controller | | controller.nodeSelector | object | `{}` | Node selectors to schedule to nodes with labels. | | controller.replicas | int | `1` | | | controller.resources.limits.cpu | int | `1` | | @@ -43,7 +43,7 @@ You can follow the detailed installation instruction [here](https://karpenter.sh | webhook.affinity | object | `{}` | Affinity rules for scheduling | | webhook.env | list | `[]` | List of environment items to add to the webhook | | webhook.hostNetwork | bool | `false` | Set to true if using custom CNI on EKS | -| webhook.image | string | `"public.ecr.aws/karpenter/webhook:v0.5.4@sha256:fd7dd0a3e155cb08a1c1def31258c654ac6c61b2fa8d8d25b7483688664c7de2"` | Image to use for the webhook | +| webhook.image | string | `"public.ecr.aws/karpenter/webhook:v0.5.5@sha256:682c4b22dae952f60c0514ebc57814fcf7d6d06967d33a678b6b66d0c04170cd"` | Image to use for the webhook | | webhook.nodeSelector | object | `{}` | Node selectors to schedule to nodes with labels. | | webhook.port | int | `8443` | | | webhook.replicas | int | `1` | | @@ -53,5 +53,3 @@ You can follow the detailed installation instruction [here](https://karpenter.sh | webhook.resources.requests.memory | string | `"50Mi"` | | | webhook.tolerations | list | `[]` | Tolerations to schedule to nodes with taints. | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.5.0](https://github.com/norwoodj/helm-docs/releases/v1.5.0) \ No newline at end of file diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 011401d82e75..099d940795ef 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -1,7 +1,6 @@ # -- Additional labels to add into metadata additionalLabels: {} # app: karpenter - serviceAccount: # -- Create a service account for the application controller create: true @@ -21,7 +20,7 @@ controller: # -- Affinity rules for scheduling affinity: {} # -- Image to use for the Karpenter controller - image: "public.ecr.aws/karpenter/controller:v0.5.4@sha256:19ebf83d64fa41d75fb19ae5047f54b1c66423b4ab5ceef36ae99a0daaad1895" + image: "public.ecr.aws/karpenter/controller:v0.5.5@sha256:f955fbf012f200048f022f361d48dd83e9ef83d61faa88086f0377cade0492ad" # -- Cluster name clusterName: "" # -- Cluster endpoint @@ -44,7 +43,7 @@ webhook: # -- Affinity rules for scheduling affinity: {} # -- Image to use for the webhook - image: "public.ecr.aws/karpenter/webhook:v0.5.4@sha256:fd7dd0a3e155cb08a1c1def31258c654ac6c61b2fa8d8d25b7483688664c7de2" + image: "public.ecr.aws/karpenter/webhook:v0.5.5@sha256:682c4b22dae952f60c0514ebc57814fcf7d6d06967d33a678b6b66d0c04170cd" # -- Set to true if using custom CNI on EKS hostNetwork: false port: 8443 diff --git a/website/config.yaml b/website/config.yaml index 4e9642853ec3..3bc26e22615f 100644 --- a/website/config.yaml +++ b/website/config.yaml @@ -65,8 +65,9 @@ params: url: 'https://slack.k8s.io/' icon: fab fa-slack desc: 'Chat with us on Slack in the #aws-provider channel' - latest_release_version: v0.5.3 + latest_release_version: v0.5.5 versions: + - "v0.5.5" - "v0.5.3" - "v0.5.2" - "v0.5.0" diff --git a/website/content/en/v0.5.5/AWS/_index.md b/website/content/en/v0.5.5/AWS/_index.md new file mode 100644 index 000000000000..13f1c3f9e722 --- /dev/null +++ b/website/content/en/v0.5.5/AWS/_index.md @@ -0,0 +1,5 @@ +--- +title: "AWS" +linkTitle: "AWS" +weight: 70 +--- \ No newline at end of file diff --git a/website/content/en/v0.5.5/AWS/launch-templates.md b/website/content/en/v0.5.5/AWS/launch-templates.md new file mode 100644 index 000000000000..aab224a80197 --- /dev/null +++ b/website/content/en/v0.5.5/AWS/launch-templates.md @@ -0,0 +1,231 @@ +--- +title: "Launch Templates and Custom Images" +linkTitle: "Launch Templates" +weight: 80 +--- + +By default, Karpenter generates launch templates that use [EKS Optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) for nodes. Often, users need to customize the node image to integrate with existing infrastructure or meet compliance requirements. Karpenter supports custom node images through Launch Templates. If you need to customize the node, then you need a custom launch template. + +Note: By customizing the image, you are taking responsibility for maintaining the image, including security updates. In the default configuration, Karpenter will use the latest version of the EKS optimized AMI, which is maintained by AWS. + + +## Introduction + +Karpenter follows existing AWS patterns for customizing the base image of +instances. More specifically, Karpenter uses [EC2 launch templates](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchTemplates.html). Launch +templates may specify many values. The pivotal value is the base image (AMI). +Launch templates further specify many different parameters related to networking, authorization, instance type, and more. + +Launch Templates and AMIs are unique to AWS regions, similar to EKS clusters. IAM resources are global. + +**Karpenter only implements a subset of launch template fields, and some fields should not be set.** + +This guide describes requirements for using launch templates with Karpenter, and later an example procedure. + +## Launch Template Requirements + +The Launch Template resource includes a large number of fields. AWS accepts launch templates with any subset of these fields defined. + +Certain fields are obviously critical, such as AMI and User Data. Some fields are useful for particular workloads, such as storage and IAM Instance Profile. + +Finally, **the majority of Launch Template fields should not be set** (or will have no effect), such as network interfaces and instance type. + +## Important Fields + +When creating a custom launch template, the AMI and User Data are the defining characteristics. Instance Profile (IAM Role) and Security Group (firewall rules) are also important for Karpenter. + +### AMI + +AMI (Amazon Machine Image), is the base image/VM for a launch template. + +[Review the instructions for importing a VM to AWS.](https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html) Note the AMI id generated by this process, such as, +`ami-074cce78125f09d61`. + +### User Data - Autoconfigure + +Importantly, the AMI must support automatically connecting to a cluster based +on "user data", or a base64 encoded string passed to the instance at startup. +The syntax and purpose of the user data varies between images. The Karpenter +default OS, Amazon Linux 2 (AL2), accepts shell scripts (bash commands). + +[AWS calls data passed to an instance at launch time "user +data".](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html#user-data-shell-scripts) + +In the default configuration, Karpenter uses an EKS optimized version of AL2 and passes the hostname of the Kubernetes API server, and a certificate. The EKS Optimized AMI includes a `bootstrap.sh` script which connects the instance to the cluster, based on the passed data. + +Alternatively, you may reference AWS's [`bootstrap.sh` +file](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) +when building a custom base image. + +``` +#!/bin/bash +/etc/eks/bootstrap.sh \ +--kubelet-extra-args <'--max-pods=40'> \ +--b64-cluster-ca \ +--apiserver-endpoint \ +--dns-cluster-ip \ +--use-max-pods false +``` + +Note, you must populate this command with live values. Karpenter will +not change the user data in the launch template. + +Encode using yaml function `!Base64` yaml function or `cat userdata.sh | base64 > userdata-encoded.txt` shell command. + +**Bootstrap Script Parameters** + +The sample bootstrap script requires information to join the cluster. + +These values may be found using: +``` +aws eks describe-cluster --name MyKarpenterCluster +``` + +**Kubelet Arguments** + +Specifying max-pods can break Karpenter's binpacking logic (it has no way to know what this setting is). If Karpenter attempts to pack more than this number of pods, the instance may be oversized, and additional pods will reschedule. + +## Situational Fields + +Configure these values in response to a particular use case, such as nodes interacting with another AWS service, or using EBS storage on the node. + +### Instance Profile - IAM + +The launch template must include an "instance profile" -- an IAM role. + +The instance profile must include *at least* the permissions of the default Karpenter node instance profile. See the default role, `KarpenterNodeRole`, in the full example below for more information. + +See also, [the managed policy "AmazonEKSWorkerNodePolicy"](https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonEKSWorkerNodePolicy) which includes permission to describe clusters and subnets. + +### Storage + +Karpenter expects nothing of node storage. Configure as needed for your base +image. + +### Security Groups - Firewall + +The launch template may include a security group (i.e., instance firewall rules) and the security group must be associated with the virtual private cloud (VPC) of the EKS cluster. If none is specified, the default security group of the cluster VPC is used. + +The security group must permit communication with EKS control plane. Outbound access should be permitted for at least: HTTPS on port 443, DNS (UDP and TCP) on port 53, and your subnet's network access control list (network ACL). + +## Fields with Undefined Behavior + +Resources referenced by these fields are controlled by EKS/Karpenter, and not the launch template. + +### Instance Type + +The instance type should not be specified in the launch template. Karpenter +will determine the launch template at run time. + +### Network Interfaces + +The [AWS CNI](https://docs.aws.amazon.com/eks/latest/userguide/pod-networking.html) will configure the network interfaces. Do not configure network instances in the launch template. + +## Creating the Launch Template + +Launch Templates may be created via the web console, the AWS CLI, or +CloudFormation. + +### CloudFormation + +The procedure, in summary, is to: +1. [Create an AMI as described in the EC2 documentation.](https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html) +2. Write a EC2 Launch Template specification including the AMI. +3. Push the specification to AWS with CloudFormation. +4. Update the Provisioner CRD to specify the new Launch Template. + +An example yaml cloudformation definition of a launch template for Karpenter is +provided below. + +CloudFormation yaml is suited for the moderately high configuration density of +launch templates, and creating the unusual InstanceProfile resource. + +You must manually replace these values in the template: +- SecurityGroupID + - list all security groups with `aws ec2 describe-security-groups` +- Parameters in UserData +- AMI + +```yaml +AWSTemplateFormatVersion: '2010-09-09' +Resources: + # create InstanceProfile wrapper on NodeRole + KarpenterNodeInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + InstanceProfileName: "KarpenterNodeInstanceProfile" + Path: "/" + Roles: + - Ref: "KarpenterNodeRole" + # create role with basic permissions for EKS node + KarpenterNodeRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: "KarpenterNodeRole" + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + !Sub "ec2.${AWS::URLSuffix}" + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + MyLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateData: + IamInstanceProfile: + # Get ARN of InstanceProfile defined above + Arn: !GetAtt + - KarpenterNodeInstanceProfile + - Arn + ImageId: ami-074cce78125f09d61 + # UserData is Base64 Encoded + UserData: !Base64 > + #!/bin/bash + /etc/eks/bootstrap.sh 'MyClusterName' \ + --kubelet-extra-args '--node-labels=node.k8s.aws/capacity-type=spot' \ + --b64-cluster-ca 'LS0t....0tCg==' \ + --apiserver-endpoint 'https://B0385BE29EA792E811CB5866D23C856E.gr7.us-east-2.eks.amazonaws.com' + BlockDeviceMappings: + - Ebs: + VolumeSize: 80 + VolumeType: gp3 + DeviceName: /dev/xvda + # The SecurityGroup must be associated with the cluster VPC + SecurityGroupIds: + - sg-a69adfdb + LaunchTemplateName: KarpenterCustomLaunchTemplate +``` + +Create the Launch Template by uploading the CloudFormation yaml file. The +sample yaml creates an IAM Object (InstanceProfile), so `--capabilities +CAPABILITY_NAMED_IAM` must be indicated. + +``` +aws cloudformation create-stack \ + --stack-name KarpenterLaunchTemplateStack \ + --template-body file://$(pwd)/lt-cfn-demo.yaml \ + --capabilities CAPABILITY_NAMED_IAM +``` + +### Define LaunchTemplate for Provisioner + +The LaunchTemplate is ready to be used. Specify it by name in the [Provisioner +CRD](../../provisioner/). Karpenter will use this template when creating new instances. + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +spec: + provider: + launchTemplate: CustomKarpenterLaunchTemplateDemo + +``` diff --git a/website/content/en/v0.5.5/AWS/provisioning.md b/website/content/en/v0.5.5/AWS/provisioning.md new file mode 100644 index 000000000000..6310073ab04c --- /dev/null +++ b/website/content/en/v0.5.5/AWS/provisioning.md @@ -0,0 +1,178 @@ +--- +title: "Provisioning Configuration" +linkTitle: "Provisioning" +weight: 10 +--- + +## spec.provider + +This section covers parameters of the AWS Cloud Provider. + +[Review these fields in the code.](https://github.com/awslabs/karpenter/blob/main/pkg/cloudprovider/aws/apis/v1alpha1/provider.go#L33) + +### InstanceProfile +An `InstanceProfile` is a way to pass a single IAM role to an EC2 instance. + +It is required, and specified by name. A suitable `KarpenterNodeRole` is created in the getting started guide. + +``` +spec: + provider: + instanceProfile: MyInstanceProfile +``` + +### LaunchTemplate + +A launch template is a set of configuration values sufficient for launching an EC2 instance (e.g., AMI, storage spec). + +A custom launch template is specified by name. If none is specified, Karpenter will automatically create a launch template. + +Review the [Launch Template documentation](../launch-templates/) to learn how to create a custom one. + +``` +spec: + provider: + launchTemplate: MyLaunchTemplate +``` + +### SubnetSelector + +Karpenter discovers subnets using [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). + +Subnets may be specified by any AWS tag, including `Name`. Selecting tag values using wildcards ("\*") is supported. + +When launching nodes, Karpenter automatically chooses a subnet that matches the desired zone. If multiple subnets exist for a zone, one is chosen randomly. + +**Examples** + +Select all subnets with a specified tag: +``` + subnetSelector: + kubernetes.io/cluster/MyCluster: '*' +``` + +Select subnets by name: +``` + subnetSelector: + Name: subnet-0fcd7006b3754e95e +``` + +Select subnets by an arbitrary AWS tag key/value pair: +``` + subnetSelector: + MySubnetTag: value +``` + +Select subnets using wildcards: +``` + subnetSelector: + Name: *public* + +``` + +### SecurityGroupSelector + +The security group of an instance is comparable to a set of firewall rules. +If no security groups are explicitly listed, Karpenter discovers them using the tag "kubernetes.io/cluster/MyClusterName", similar to subnet discovery. + +EKS creates at least two security groups by default, [review the documentation](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) for more info. + +Security groups may be specified by any AWS tag, including "name". Selecting tags using wildcards ("*") is supported. + +‼️ When launching nodes, Karpenter uses all of the security groups that match the selector. If multiple security groups with the tag `kubernetes.io/cluster/MyClusterName` match the selector, this may result in failures using the AWS Load Balancer controller. The Load Balancer controller only supports a single security group having that tag key. See this [issue](https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2367) for more details. + +To verify if this restriction affects you, run the following commands. +```bash +CLUSTER_VPC_ID="$(aws eks describe-cluster --name $CLUSTER_NAME --query cluster.resourcesVpcConfig.vpcId --output text)" + +aws ec2 describe-security-groups --filters Name=vpc-id,Values=$CLUSTER_VPC_ID Name=tag-key,Values=kubernetes.io/cluster/$CLUSTER_NAME --query SecurityGroups[].[GroupName] --output text +``` + +If multiple securityGroups are printed, you will need a more targeted securityGroupSelector. + +**Examples** + +Select all security groups with a specified tag: +``` +spec: + provider: + securityGroupSelector: + kubernetes.io/cluster/MyKarpenterSecurityGroups: '*' +``` + +Select security groups by name, or another tag (all criteria must match): +``` + securityGroupSelector: + Name: sg-01077157b7cf4f5a8 + MySecurityTag: '' # matches all resources with the tag +``` + +Select security groups by name using a wildcard: +``` + securityGroupSelector: + Name: *public* +``` + +### Tags + +Tags will be added to every EC2 Instance launched by this provisioner. + +``` +spec: + provider: + tags: + InternalAccountingTag: 1234 + dev.corp.net/app: Calculator + dev.corp.net/team: MyTeam +``` +Note: Karpenter will set the default AWS tags listed below, but these can be overridden in the tags section above. +``` +Name: karpenter.sh/cluster//provisioner/ +karpenter.sh/cluster/: owned +kubernetes.io/cluster/: owned +``` + +### Metadata Options + +Control the exposure of [Instance Metadata Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) on EC2 Instances launched by this provisioner using a generated launch template. + +Refer to [recommended, security best practices](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. + +If metadataOptions are omitted from this provisioner, the following default settings will be used. + +``` +spec: + provider: + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required +``` + + +## Other Resources + +### Accelerators, GPU + +Accelerator (e.g., GPU) values include +- `nvidia.com/gpu` +- `amd.com/gpu` +- `aws.amazon.com/neuron` + +Karpenter supports accelerators, such as GPUs. + + +Additionally, include a resource requirement in the workload manifest. This will cause the GPU dependent pod will be scheduled onto the appropriate node. + +*Accelerator resource in workload manifest (e.g., pod)* + +```yaml +spec: + template: + spec: + containers: + - resources: + limits: + nvidia.com/gpu: "1" +``` diff --git a/website/content/en/v0.5.5/_index.md b/website/content/en/v0.5.5/_index.md new file mode 100755 index 000000000000..fd0addc59a24 --- /dev/null +++ b/website/content/en/v0.5.5/_index.md @@ -0,0 +1,27 @@ + +--- +title: "Documentation" +linkTitle: "Docs" +weight: 20 +cascade: + type: docs +--- +Karpenter is an open-source node provisioning project built for Kubernetes. +Adding Karpenter to a Kubernetes cluster can dramatically improve the efficiency and cost of running workloads on that cluster. +Karpenter is tightly integrated with Kubernetes features to make sure that the right types and amounts of compute resources are available to pods as they are needed. +Karpenter works by: + +* **Watching** for pods that the Kubernetes scheduler has marked as unschedulable +* **Evaluating** scheduling constraints (resource requests, nodeselectors, affinities, tolerations, and topology spread constraints) requested by the pods +* **Provisioning** nodes that meet the requirements of the pods +* **Scheduling** the pods to run on the new nodes +* **Removing** the nodes when the nodes are no longer needed + +As a cluster administrator, you can configure an unconstrained Karpenter provisioner when it is first installed and not change it again. +Other times, you might continue to tweak the provisioner or create multiple provisioners for a cluster used by different teams. +On-going cluster administrator tasks include upgrading and decommissioning nodes. + +As an application developer, you can make specific requests for capacity and features you want from the nodes running your pods. +Karpenter is designed to quickly create the best possible nodes to meet those needs and schedule the pods to run on them. + +Learn more about Karpenter and how to get started below. diff --git a/website/content/en/v0.5.5/concepts/_index.md b/website/content/en/v0.5.5/concepts/_index.md new file mode 100644 index 000000000000..a84e912ff2d1 --- /dev/null +++ b/website/content/en/v0.5.5/concepts/_index.md @@ -0,0 +1,164 @@ +--- +title: "Concepts" +linkTitle: "Concepts" +weight: 35 +--- + +Users fall under two basic roles: Kubernetes cluster administrators and application developers. +This document describes Karpenter concepts through the lens of those two types of users. + +## Cluster administrator + +As a Kubernetes cluster administrator, you can engage with Karpenter to: + +* Install Karpenter +* Configure provisioners to set constraints and other features for managing nodes +* Deprovision nodes +* Upgrade nodes + +Concepts associated with this role are described below. + + +### Installing Karpenter + +Karpenter is designed to run on a node in your Kubernetes cluster. +As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. + +[Getting Started with Karpenter on AWS]{{< ref "../getting-started" >}}) +describes the process of installing Karpenter on an AWS cloud provider. +Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. +For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. + +Once privileges are in place, Karpenter is deployed with a Helm chart. + +### Configuring provisioners + +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. +To configure Karpenter, you create *provisioners* that define how Karpenter manages unschedulable pods and expires nodes. +Here are some things to know about the Karpenter provisioner: + +* **Unschedulable pods**: Karpenter only attempts to schedule pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. + +* **Provisioner CR**: Karpenter defines a Custom Resource called a Provisioner to specify provisioning configuration. +Each provisioner manages a distinct set of nodes, but pods can be scheduled to any provisioner that supports its scheduling constraints. +A provisioner contains constraints that impact the nodes that can be provisioned and attributes of those nodes (such timers for removing nodes). +See [Provisioner API](../provisioner) for a description of settings and the [Provisioning](../tasks/provisioning) task for provisioner examples. + +* **Well-known labels**: The provisioner can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. +See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. +Keep in mind that only a subset of these labels are supported in Karpenter, as described later. + +* **Deprovisioning nodes**: A provisioner can also include time-to-live values to indicate when nodes should be deprovisioned after a set amount of time from when they were created or after they becomes empty of deployed pods. + +* **Multiple provisioners**: Multiple provisioners can be configured on the same cluster. +For example, you might want to configure different teams on the same cluster to run on completely separate capacity. +One team could run on nodes nodes using BottleRocket, while another uses EKSOptimizedAMI. + +Although most use cases are addressed with a single provisioner for multiple teams, multiple provisioners are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different deprovisioning settings. + +### Deprovisioning nodes + +Karpenter deletes nodes when they are no longer needed. + +* **Finalizer**: Karpenter places a finalizer bit on each node it creates. +When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. +Karpenter handles all clean-up work needed to properly delete the node. +* **Node Expiry**: If a node expiry time-to-live value (`ttlSecondsUntilExpired`) is reached, that node is drained of pods and deleted (even if it is still running workloads). +* **Empty nodes**: When the last workload pod running on a Karpenter-managed node is gone, the node is annotated with an emptiness timestamp. +Once that "node empty" time-to-live (`ttlSecondsAfterEmpty`) is reached, finalization is triggered. + +For more details on how Karpenter deletes nodes, see [Deprovisioning nodes](../tasks/deprovisioning) for details. + +### Upgrading nodes + +A straight-forward way to upgrade nodes is to set `ttlSecondsUntilExpired`. +Nodes will be terminated after a set period of time and will be replaced with newer nodes. + +Understanding the following concepts will help you in carrying out the tasks just described. + +### Constraints + +The concept of layered constraints is key to using Karpenter. +With no constraints defined in provisioners and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. +Nodes can be created using any instance type and run in any zones. + +An application developer can tighten the constraints defined in a provisioner by the cluster administrator by defining additional scheduling constraints in their pod spec. +Refer to the description of Karpenter constraints in the Application Developer section below for details. + +### Scheduling + +Karpenter schedules pods that the Kubernetes scheduler has marked unschedulable. +After solving scheduling constraints and launching capacity, Karpenter optimistically creates the Node object and binds the pod. +This stateless approach helps to avoid race conditions and improves performance. +If something is wrong with the launched node, Kubernetes will automatically migrate the pods to a new node. + +Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. +This is useful if there is additional room in the node due to imperfect packing shape or because workloads finish over time. + +### Cloud provider +Karpenter makes requests to provision new nodes to the associated cloud provider. +The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. +Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. + +While using Kubernetes well-known labels, the provisioner can set some values that are specific to the cloud provider. +So, for example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). + +### Kubernetes cluster autoscaler +Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is +designed to add nodes when requests come in to run pods that cannot be met by current capacity. +Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. +By taking a fresh look at provisioning, Karpenter offers the following improvements: + +* **Designed to handle the full flexibility of the cloud**: +Karpenter has the ability to efficiently address the full range of instance types available through AWS. +Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. + +* **Group-less node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. +This enables it to retry in milliseconds instead of minutes when capacity is unavailable. +It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. + +* **Scheduling enforcement**: Cluster autoscaler doesn’t bind pods to the nodes it creates. +Instead, it relies on the kube-scheduler to make the same scheduling decision after the node has come online. +A node that Karpenter launches has its pods bound immediately. +The kubelet doesn't have to wait for the scheduler or for the node to become ready. +It can start preparing the container runtime immediately, including pre-pulling the image. +This can shave seconds off of node startup latency. + +## Application developer + +As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. +Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. +These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). + +Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. +A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). +The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. + +The Kubernetes scheduler tries to match those constraints with available nodes. +If the pod is unschedulable, Karpenter creates compute resources that match its needs. +When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. + +As long as the requests are not outside of the provisioner's constraints, +Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. +Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. + +So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? + +Kubernetes features that Karpenter supports for scheduling pods include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). +It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) and [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). + +From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, +you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. +Those that are implemented in Karpenter include: + +* **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 +* **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium +* **topology.kubernetes.io/zone**: For example, topology.kubernetes.io/zone=us-east-1c + +{{% alert title="Note" color="primary" %}} +Don't use `podAffinity` and `podAntiAffinity` to schedule pods on the same or different nodes as other pods. +Kubernetes SIG scalability recommends against these features due to their negative performance impact on the Kubernetes Scheduler (see [KEP 895](https://github.com/kubernetes/enhancements/tree/master/keps/sig-scheduling/895-pod-topology-spread#impact-to-other-features)) and Karpenter doesn't support them for the moment (you can follow their consideration by subscribing to the [issue](https://github.com/aws/karpenter/issues/942). +Instead, the Karpenter project recommends `topologySpreadConstraints` to reduce blast radius and `nodeSelectors` and `taints` to implement colocation. +{{% /alert %}} + +For more on how, as a developer, you can add constraints to your pod deployment, see [Scheduling](../tasks/scheduling/) for details. diff --git a/website/content/en/v0.5.5/development-guide.md b/website/content/en/v0.5.5/development-guide.md new file mode 100644 index 000000000000..2935246847a0 --- /dev/null +++ b/website/content/en/v0.5.5/development-guide.md @@ -0,0 +1,102 @@ +--- +title: "Development Guide" +linkTitle: "Development Guide" +weight: 80 +--- + +## Dependencies + +The following tools are required for contributing to the Karpenter project. + +| Package | Version | Install | +| ------------------------------------------------------------------ | -------- | ---------------------------------------------- | +| [go](https://golang.org/dl/) | v1.15.3+ | [Instructions](https://golang.org/doc/install) | +| [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) | | `brew install kubectl` | +| [helm](https://helm.sh/docs/intro/install/) | | `brew install helm` | +| Other tools | | `make toolchain` | + +## Developing + +### Setup / Teardown + +Based on how you are running your Kubernetes cluster, follow the [Environment specific setup](#environment-specific-setup) to configure your environment before you continue. Once you have your environment set up, to install Karpenter in the Kubernetes cluster specified in your `~/.kube/config` run the following commands. + +``` +CLOUD_PROVIDER= make apply # Install Karpenter +make delete # Uninstall Karpenter +``` + +### Developer Loop +* Make sure dependencies are installed + * Run `make codegen` to make sure yaml manifests are generated + * Run `make toolchain` to install cli tools for building and testing the project +* You will need a personal development image repository (e.g. ECR) + * Make sure you have valid credentials to your development repository. + * `$KO_DOCKER_REPO` must point to your development repository + * Your cluster must have permissions to read from the repository +* It's also a good idea to persist `$CLOUD_PROVIDER` in your environment variables to simplify the `make apply` command. + +### Build and Deploy +*Note: these commands do not rely on each other and may be executed independently* +```sh +make apply # quickly deploy changes to your cluster +make dev # run codegen, lint, and tests +``` + +### Testing +```sh +make test # E2e correctness tests +make battletest # More rigorous tests run in CI environment +``` + +### Verbose Logging +```sh +kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' +``` + +### Debugging Metrics +OSX: +```sh +open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 +``` + +Linux: +```sh +gio open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 +``` + +### Tailing Logs +While you can tail Karpenter's logs with kubectl, there's a number of tools out there that enhance the experience. We recommend [Stern](https://pkg.go.dev/github.com/planetscale/stern#section-readme): + +```sh +stern -l karpenter=controller -n karpenter +``` + +## Environment specific setup + +### AWS +Set the CLOUD_PROVIDER environment variable to build cloud provider specific packages of Karpenter. + +```sh +export CLOUD_PROVIDER=aws +``` + +For local development on Karpenter you will need a Docker repo which can manage your images for Karpenter components. +You can use the following command to provision an ECR repository. +```sh +aws ecr create-repository \ + --repository-name karpenter/controller \ + --image-scanning-configuration scanOnPush=true \ + --region ${AWS_DEFAULT_REGION} +aws ecr create-repository \ + --repository-name karpenter/webhook \ + --image-scanning-configuration scanOnPush=true \ + --region ${AWS_DEFAULT_REGION} +``` + +Once you have your ECR repository provisioned, configure your Docker daemon to authenticate with your newly created repository. + +```sh +export KO_DOCKER_REPO="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/karpenter" +aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin $KO_DOCKER_REPO +``` diff --git a/website/content/en/v0.5.5/faq.md b/website/content/en/v0.5.5/faq.md new file mode 100644 index 000000000000..a739e6dfdf1c --- /dev/null +++ b/website/content/en/v0.5.5/faq.md @@ -0,0 +1,129 @@ +--- +title: "FAQs" +linkTitle: "FAQs" +weight: 90 +--- +## General + +### How does a provisioner decide to manage a particular node? +See [Configuring provisioners]({{< ref "./concepts/#configuring-provisioners" >}}) for information on how Karpenter provisions and manages nodes. + +### What cloud providers are supported? +AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. +See [Cloud provider]({{< ref "./concepts/#cloud-provider" >}}) for details. + +### Can I write my own cloud provider for Karpenter? +Yes, but there is no documentation yet for it. +Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter/tree/main/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. + +### What operating system nodes does Karpenter deploy? +By default, Karpenter uses Amazon Linux 2 images. + +### Can I provide my own custom operating system images? +Karpenter allows you to create your own AWS AMIs using custom launch templates. +See [Launch Templates and Custom Images]({{< ref "./aws/launch-templates/" >}}) for details. + +### Can Karpenter deal with workloads for mixed architecture cluster (arm vs. amd)? +Yes. Build and prepare custom arm images as described in [Launch Templates and Custom Images]({{< ref "./aws/launch-templates/" >}}). +Specify the desired architecture when you deploy workloads. + +### What RBAC access is required? +All of the required RBAC rules can be found in the helm chart template. +See the [rbac.yaml](https://github.com/aws/karpenter/blob/main/charts/karpenter/templates/controller/rbac.yaml) file for details. + +### Can I run Karpenter outside of a Kubernetes cluster? +Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. + +## Compatibility + +### Which versions of Kubernetes does Karpenter support? +Karpenter is tested with Kubernetes v1.19 and later. + +### What Kubernetes distributions are supported? +Karpenter documents integration with a fresh install of the latest AWS Elastic Kubernetes Service (EKS). +Existing EKS distributions can be used, but this use case has not yet been documented. +Other Kubernetes distributions (KOPs, etc.) can be used, but setting up cloud provider permissions for those distributions has not been documented. + +### How does Karpenter interact with AWS node group features? +Provisioners are designed to work alongside static capacity management solutions like EKS Managed Node Groups and EC2 Auto Scaling Groups. +You can manage all capacity using provisioners, use a mixed model with dynamic and statically managed capacity, or use a fully static approach. +We expect most users will use a mixed approach in the near term and provisioner-managed in the long term. + + +### How does Karpenter interact with Kubernetes features? +* Kubernetes Cluster Autoscaler: Karpenter can work alongside cluster autoscaler. +See [Kubernetes cluster autoscaler]({{< ref "./concepts/#kubernetes-cluster-autoscaler" >}}) for details. +* Kubernetes Scheduler: Karpenter focuses on scheduling pods that the Kubernetes scheduler has marked as unschedulable. +See [Scheduling]({{< ref "./concepts/#scheduling" >}}) for details on how Karpenter interacts with the Kubernetes scheduler. + +## Provisioning +### What features does the Karpenter provisioner support? +See [Provisioner API]({{< ref "./provisioner" >}}) for provisioner examples and descriptions of features. + +### Can I create multiple (team-based) provisioners on a cluster? +Yes, provisioners can identify multiple teams based on labels. +See [Provisioner API]({{< ref "./provisioner" >}}) for details. + +### If multiple provisioners are defined, which will my pod use? + +By default, pods will use the rules defined by a provisioner named default. +This is analogous to the default scheduler. +To select an alternative provisioner, use the node selector `karpenter.sh/provisioner-name: alternative-provisioner`. +You must either define a default provisioner or explicitly specify `karpenter.sh/provisioner-name node selector`. + +### Can I set total limits of CPU and memory for a provisioner? +Yes, the setting is provider-specific. +See examples in [Accelerators, GPU]({{< ref "./aws/provisioning/#accelerators-gpu" >}}) Karpenter documentation. + +### Can I mix spot and on-demand EC2 run types? +Yes, see [Example Provisioner Resource]({{< ref "./provisioner/#example-provisioner-resource" >}}) for an example. + +### Can I restrict EC2 instance types? + +* Attribute-based requests are currently not possible. +* You can select instances with special hardware, such as gpu. + +### How does Karpenter dynamically select instance types? + +Karpenter batches pending pods and then binpacks them based on CPU, memory, and GPUs required, taking into account node overhead, VPC CNI resources required, and daemon sets that will be packed when bringing up a new node. +By default Karpenter uses all available instance types, but it can be constrained in the provisioner spec with the [instance-type](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type) well-known label in the requirements section. +After the pods are binpacked on the most efficient instance type (i.e. the smallest instance type that can fit the pod batch), Karpenter takes 19 other instance types that are larger than the most efficient packing, and passes all 20 instance type options to an API called Amazon EC2 Fleet. +The EC2 fleet API attempts to provision the instance type based on a user-defined allocation strategy. +If you are using the on-demand capacity type, then Karpenter uses the `lowest-price` allocation strategy. +So fleet will provision the lowest price instance type it can get from the 20 Karpenter passed it. +If the instance type is unavailable for some reason, then fleet will move on to the next cheapest instance type. +If you are using the spot capacity type, Karpenter uses the capacity-optimized-prioritized allocation strategy which tells fleet to find the instance type that EC2 has the most capacity of which will decrease the probability of a spot interruption happening in the near term. +See [Choose the appropriate allocation strategy](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-allocation-strategy.html#ec2-fleet-allocation-use-cases) for information on fleet optimization. + +## Workloads + +### How can someone deploying pods take advantage of Karpenter? + +See [Application developer]({{< ref "./concepts/#application-developer" >}}) for descriptions of how Karpenter matches nodes with pod requests. + +### How do I use Karpenter with the AWS load balancer controller? + +* Set the [ALB target type](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.3/guide/ingress/annotations/#target-type) to IP mode for the pods. +Use IP targeting if you want the pods to receive equal weight. +Instance balancing could greatly skew the traffic being sent to a node without also managing host spread of the workload. +* Set [readiness gate](https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.3/deploy/pod_readiness_gate/) on the namespace. +The default is round robin at the node level. +For Karpenter, not all nodes are equal. +For example, each node will have different performance characteristics and a different number of pods running on it. +A `t3.small` with three instances should not receive the same amount of traffic as a `m5.4xlarge` with dozens of pods. +If you don't specify a spread at the workload level, or limit what instances should be picked, you could get the same amount of traffic sent to the `t3` and `m5`. + +### Can I use Karpenter with EBS disks per availability zone? +Not yet. + +### Can I set `--max-pods` on my nodes? +Not yet. + +## Deprovisioning +### How does Karpenter deprovision nodes? +See [Deprovisioning nodes]({{< ref "./tasks/deprovisioning" >}}) for information on how Karpenter deprovisions nodes. + +## Upgrading +### How do I upgrade Karpenter? +Karpenter is a controller that runs in your cluster, but it is not tied to a specific Kubernetes version, as the Cluster Autoscaler is. +Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes and keep Karpenter up to date on bug fixes and new features. diff --git a/website/content/en/v0.5.5/getting-started-with-kops/_index.md b/website/content/en/v0.5.5/getting-started-with-kops/_index.md new file mode 100644 index 000000000000..d786e5194765 --- /dev/null +++ b/website/content/en/v0.5.5/getting-started-with-kops/_index.md @@ -0,0 +1,161 @@ + +--- +title: "Getting Started with kOps" +linkTitle: "Getting Started with kOps" +weight: 10 +--- + +In this example, the cluster will be running on Amazon Web Services (AWS) managed by [kOps](https://kops.sigs.k8s.io/). +Karpenter is designed to be cloud provider agnostic, but currently only supports AWS. Contributions are welcomed + +Karpenter is supported on kOps as of 1.24.0-alpha.2, but sits behind a feature flag as the interface between kOps and Karpenter is +still work in progress and is likely to change significantly. This guide is intended for users that wants to test Karpenter on kOps and provide feedback to Karpenter and kOps developers. +Read more about how Karpenter works on kOps and the current limitations in the [kOPs Karpenter documentation](https://kops.sigs.k8s.io/operations/karpenter/). + +This guide should take less than 1 hour to complete, and cost less than $0.25. +Follow the clean-up instructions to reduce any charges. + +This guide assumes you already have a kOps state store and a hosted zone. If you do not have one, +run through the [kOps getting started on AWS documentation](https://kops.sigs.k8s.io/getting_started/aws/) up until "Creating your first cluster". + +## Install + +Karpenter is installed in clusters as a managed addon. kOps will automatically create +and manage the necessary the IAM roles and policies Karpenter needs. + +### Required Utilities + +Install these tools before proceeding: + +1. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) +2. `kops` - [kubectl, but for clusters](https://kops.sigs.k8s.io/getting_started/install/) v1.24.0 or later + +### Environment Variables + +After setting up the tools, set the following environment variables used by kOps. + +```bash +export KOPS_FEATURE_FLAGS=Karpenter +export CLUSTER_NAME=${USER}-karpenter-demo.example.com +export ZONES=us-west-2a +export KOPS_STATE_STORE=s3://prefix-example-com-state-store +export KOPS_OIDC_STORE=s3://prefix-example-com-oidc-store/discovery +``` + +### Create a Cluster + +kOps installs Karpenter on the control plane. Once the control plane is running, Karpenter will provision the +the worker nodes needed for non-Control Plane Deployments such as CoreDNS and CSI drivers. + +The following command will launch a cluster with Karpenter-managed worker nodes: + +```bash +kops create cluster \ + --zones=$ZONES \ + --discovery-store=${KOPS_OIDC_STORE} \ + --instance-manager=karpenter \ + --networking=amazonvpc \ + ${CLUSTER_NAME} \ + --yes +``` + +Note: we are using AWS VPC CNI for networking as Karpenter's binpacking logic assumes ENI-based networking. + +### Provisioner + +A single Karpenter provisioner is capable of handling many different pod +shapes. Karpenter makes scheduling and provisioning decisions based on pod +attributes such as labels and affinity. In other words, Karpenter eliminates +the need to manage many different InstanceGroups. + +kOps manage provisioners through InstanceGroups. Your cluster will already have +one Provisioner that will contain a suitable set of instance types for Karpenter to +choose from. + +Managing Provisioner resources directly is possible, but not straight-forward. Read +more about managing provisioners in the [kOPs Karpenter documentation](https://kops.sigs.k8s.io/operations/karpenter/) + +## First Use + +Karpenter is now active and ready to begin provisioning nodes. +As mentioned above, you should already have some Karpenter-managed nodes in your cluster used by +other kOps addons. Create additional pods using a Deployment, and watch Karpenter provision nodes in response. + +### Automatic Node Provisioning + +This deployment uses the [pause image](https://www.ianlewis.org/en/almighty-pause-container) and starts with zero replicas. + +```bash +cat <}}" + + set { + name = "serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn" + value = module.iam_assumable_role_karpenter.iam_role_arn + } + + set { + name = "controller.clusterName" + value = var.cluster_name + } + + set { + name = "controller.clusterEndpoint" + value = module.eks.cluster_endpoint + } +} +``` + +Now, deploy Karpenter by applying the new Terraform config. + +```bash +terraform init +terraform apply -var cluster_name=$CLUSTER_NAME +``` + + +### Enable Debug Logging (optional) +```sh +kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' +``` + +### Provisioner + +A single Karpenter provisioner is capable of handling many different pod +shapes. Karpenter makes scheduling and provisioning decisions based on pod +attributes such as labels and affinity. In other words, Karpenter eliminates +the need to manage many different node groups. + +Create a default provisioner using the command below. This provisioner +configures instances to connect to your cluster's endpoint and discovers +resources like subnets and security groups using the cluster's name. + +The `ttlSecondsAfterEmpty` value configures Karpenter to terminate empty nodes. +This behavior can be disabled by leaving the value undefined. + +Review the [provisioner CRD](../provisioner) for more information. For example, +`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. + +Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. + +```bash +cat < cluster.yaml +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: ${CLUSTER_NAME} + region: ${AWS_DEFAULT_REGION} + version: "1.21" + tags: + karpenter.sh/discovery: ${CLUSTER_NAME} +managedNodeGroups: + - instanceType: m5.large + amiFamily: AmazonLinux2 + name: ${CLUSTER_NAME}-ng + desiredCapacity: 1 + minSize: 1 + maxSize: 10 +iam: + withOIDC: true +EOF +eksctl create cluster -f cluster.yaml +``` + +This guide uses a managed node group to host Karpenter. + +Karpenter itself can run anywhere, including on [self-managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/worker.html), [managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html), or [AWS Fargate](https://aws.amazon.com/fargate/). + +Karpenter will provision EC2 instances in your account. + +### Create the KarpenterNode IAM Role + +Instances launched by Karpenter must run with an InstanceProfile that grants permissions necessary to run containers and configure networking. Karpenter discovers the InstanceProfile using the name `KarpenterNodeRole-${ClusterName}`. + +First, create the IAM resources using AWS CloudFormation. + +```bash +TEMPOUT=$(mktemp) + +curl -fsSL https://karpenter.sh{{< relref "." >}}cloudformation.yaml > $TEMPOUT \ +&& aws cloudformation deploy \ + --stack-name Karpenter-${CLUSTER_NAME} \ + --template-file ${TEMPOUT} \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides ClusterName=${CLUSTER_NAME} +``` + +Second, grant access to instances using the profile to connect to the cluster. This command adds the Karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. + +```bash +eksctl create iamidentitymapping \ + --username system:node:{{EC2PrivateDNSName}} \ + --cluster ${CLUSTER_NAME} \ + --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} \ + --group system:bootstrappers \ + --group system:nodes +``` + +Now, Karpenter can launch new EC2 instances and those instances can connect to your cluster. + +### Create the KarpenterController IAM Role + +Karpenter requires permissions like launching instances. This will create an AWS IAM Role, Kubernetes service account, and associate them using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html). + +``` +eksctl create iamserviceaccount \ + --cluster $CLUSTER_NAME --name karpenter --namespace karpenter \ + --attach-policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/KarpenterControllerPolicy-$CLUSTER_NAME \ + --approve +``` + +### Create the EC2 Spot Service Linked Role + +This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). +```bash +aws iam create-service-linked-role --aws-service-name spot.amazonaws.com +# If the role has already been successfully created, you will see: +# An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name AWSServiceRoleForEC2Spot has been taken in this account, please try a different suffix. +``` + +### Install Karpenter Helm Chart + +Use helm to deploy Karpenter to the cluster. + +We created a Kubernetes service account when we created the cluster using +eksctl. Thus, we don't need the helm chart to do that. + +```bash +helm repo add karpenter https://charts.karpenter.sh +helm repo update +helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ + --create-namespace --set serviceAccount.create=false --version {{< param "latest_release_version" >}} \ + --set controller.clusterName=${CLUSTER_NAME} \ + --set controller.clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` + +### Enable Debug Logging (optional) +```sh +kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' +``` + +### Create Grafana dashboards (optional) + +The Karpenter repo contains multiple [importable dashboards](https://github.com/aws/karpenter/tree/main/grafana-dashboards) for an existing Grafana instance. See the Grafana documentation for [instructions](https://grafana.com/docs/grafana/latest/dashboards/export-import/#import-dashboard) to import a dashboard. + +#### Deploy a temporary Prometheus and Grafana stack (optional) + +The following commands will deploy a Prometheus and Grafana stack that is suitable for this guide but does not include persistent storage or other configurations that would be necessary for monitoring a production deployment of Karpenter. + +```sh +helm repo add grafana-charts https://grafana.github.io/helm-charts +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update + +kubectl create namespace monitoring + +curl -fsSL https://karpenter.sh/docs/getting-started/prometheus-values.yaml | tee prometheus-values.yaml +helm install --namespace monitoring prometheus prometheus-community/prometheus --values prometheus-values.yaml + +curl -fsSL https://karpenter.sh/docs/getting-started/grafana-values.yaml | tee grafana-values.yaml +helm install --namespace monitoring grafana grafana-charts/grafana --values grafana-values.yaml +``` + +The Grafana instance may be accessed using port forwarding. + +```sh +kubectl port-forward --namespace monitoring svc/grafana 3000:80 +``` + +The new stack has only one user, `admin`, and the password is stored in a secret. The following command will retrieve the password. + +```sh +kubectl get secret --namespace monitoring grafana -o jsonpath="{.data.admin-password}" | base64 --decode +``` + +### Provisioner + +A single Karpenter provisioner is capable of handling many different pod +shapes. Karpenter makes scheduling and provisioning decisions based on pod +attributes such as labels and affinity. In other words, Karpenter eliminates +the need to manage many different node groups. + +Create a default provisioner using the command below. +This provisioner uses `securityGroupSelector` and `subnetSelector` to discover resources used to launch nodes. +We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. +Depending how these resources are shared between clusters, you may need to use different tagging schemes. + +The `ttlSecondsAfterEmpty` value configures Karpenter to terminate empty nodes. +This behavior can be disabled by leaving the value undefined. + +Review the [provisioner CRD](../provisioner) for more information. For example, +`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. + +Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. + +```bash +cat < + Provisioner API reference page +--- + +## Example Provisioner Resource + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # If nil, the feature is disabled, nodes will never expire + ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; + + # If nil, the feature is disabled, nodes will never scale down due to low utilization + ttlSecondsAfterEmpty: 30 + + # Provisioned nodes will have these taints + # Taints may prevent pods from scheduling if they are not tolerated + taints: + - key: example.com/special-taint + effect: NoSchedule + + # Labels are arbitrary key-values that are applied to all nodes + labels: + billing-team: my-team + + # Requirements that constrain the parameters of provisioned nodes. + # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # Operators { In, NotIn } are supported to enable including or excluding values + requirements: + - key: "node.kubernetes.io/instance-type" + operator: In + values: ["m5.large", "m5.2xlarge"] + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b"] + - key: "kubernetes.io/arch" + operator: In + values: ["arm64", "amd64"] + - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand + operator: In + values: ["spot", "on-demand"] + # These fields vary per cloud provider, see your cloud provider specific documentation + provider: {} +``` + +## spec.requirements + +Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the Provisioner API. + +These well known labels may be specified at the provisioner level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using the both the provisioner's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the provisioner's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. + +For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the provisioner list and the provisioner has instance type requirements, Karpenter will not create a node or schedule the pod. + +📝 None of these values are required. + +### Instance Types + +- key: `node.kubernetes.io/instance-type` + +Generally, instance types should be a list and not a single value. Leaving this field undefined is recommended, as it maximizes choices for efficiently placing pods. + +☁️ **AWS** + +Review [AWS instance types](https://aws.amazon.com/ec2/instance-types/). + +The default value includes all instance types with the exclusion of metal +(non-virtualized), +[non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html), +and GPU instances. + +View the full list of instance types with `aws ec2 describe-instance-types`. + +**Example** + +*Set Default with provisioner.yaml* + +```yaml +spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["m5.large", "m5.2xlarge"] +``` + +*Override with workload manifest (e.g., pod)* + +```yaml +spec: + template: + spec: + nodeSelector: + node.kubernetes.io/instance-type: m5.large +``` + +### Availability Zones + +- key: `topology.kubernetes.io/zone` +- value example: `us-east-1c` + +☁️ **AWS** + +- value list: `aws ec2 describe-availability-zones --region ` + +Karpenter can be configured to create nodes in a particular zone. Note that the Availability Zone `us-east-1a` for your AWS account might not have the same location as `us-east-1a` for another AWS account. + +[Learn more about Availability Zone +IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) + +### Architecture + +- key: `kubernetes.io/arch` +- values + - `amd64` (default) + - `arm64` + +Karpenter supports `amd64` nodes, and `arm64` nodes. + + +### Capacity Type + +- key: `karpenter.sh/capacity-type` + +☁️ **AWS** + +- values + - `spot` (default) + - `on-demand` + +Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). + + +## spec.kubeletConfiguration + +Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for +additional customization and use cases. Adjust these only if you know you need to do so. + +```yaml +spec: + kubeletConfiguration: + clusterDNS: ["10.0.1.100"] +``` + + +## spec.provider + +This section is cloud provider specific. Reference the appropriate documentation: + +- [AWS](../aws/provisioning/) + + + diff --git a/website/content/en/v0.5.5/reinvent.md b/website/content/en/v0.5.5/reinvent.md new file mode 100644 index 000000000000..45481c63740c --- /dev/null +++ b/website/content/en/v0.5.5/reinvent.md @@ -0,0 +1,54 @@ +# Karpenter re:Invent 2021 Builders Session +​ +![](https://github.com/aws/karpenter/raw/main/website/static/banner.png) +​ +## Prerequisites +Please install the following tools before starting: +- [AWS CLI](https://aws.amazon.com/cli/). If you're on macOS and have [Homebrew](https://brew.sh/) installed, simply `brew install awscli`. Otherwise, follow the AWS CLI [user's guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html). +- [Helm](https://helm.sh/docs/intro/install/), the Kubernetes package manager. If you're on macOS, feel free to simply `brew install helm`. Otherwise, follow the [Helm installation guide](https://helm.sh/docs/intro/install/). +​ +## Get Started +Once you have all the necessary tools installed, configure your shell with the credentials for the temporary AWS account created for this session by: +1. Navigating to the Event Engine team dashboard and clicking on the "☁️ AWS Console" button +2. Configuring your shell with the credentials required by copy and pasting the command for your operating system. +3. Running the following to set your `AWS_ACCOUNT_ID` environmental variable: + ```bash + export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" + ``` +4. Updating your local Kubernetes configuration (`kubeconfig`) by running: + ```bash + aws eks update-kubeconfig --name karpenter-demo --role-arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterEESetupRole-karpenter-demo + ``` +5. Creating an AWS [IAM service-linked role](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html) so that Karpenter can provision Spot EC2 instances with the following command: + ```bash + aws iam create-service-linked-role --aws-service-name spot.amazonaws.com + ``` + _**N.B.** If the role was created previously, you will see:_ + ```bash + # An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name AWSServiceRoleForEC2Spot has been taken in this account, please try a different suffix. + ``` +​ +If you can run the following command and see the pods running in your EKS cluster, you're all set! If not, please ask for help from one of the speakers in the session and they'll get you squared away. For your reference, the cluster name is `karpenter-demo`. +```bash +kubectl get pods -A +``` +​ +Congratulations! You now have access to an Amazon EKS cluster with an EKS Managed Node Group as well as all the AWS infrastructure necessary to use Karpenter. +Happy Building 🔨! +​ +## Install Karpenter + Use the following command to install Karpenter into your cluster: +```bash +helm repo add karpenter https://charts.karpenter.sh +helm repo update +helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ + --create-namespace --set serviceAccount.create=false --version {{< param "latest_release_version" >}} \ + --set controller.clusterName=karpenter-demo \ + --set controller.clusterEndpoint=$(aws eks describe-cluster --name karpenter-demo --query "cluster.endpoint" --output json) \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` +​ +## Next Steps +If you're a Kubernetes expert, feel free to start exploring how Karpenter works on your own and if you have any questions, one of the AWS speakers will be happy to answer them. +​ +If you'd like a guided walkthrough of Karpenter's features and capabilities, you can follow the Karpenter Getting Started guide starting at the ["Provisioner" step](https://karpenter.sh/docs/getting-started/#provisioner). Please don't hesitate to ask your AWS speaker any questions you might have! diff --git a/website/content/en/v0.5.5/tasks/_index.md b/website/content/en/v0.5.5/tasks/_index.md new file mode 100755 index 000000000000..f84a1a7cb00b --- /dev/null +++ b/website/content/en/v0.5.5/tasks/_index.md @@ -0,0 +1,7 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 45 +--- + +Karpenter tasks can be divided into those for a cluster administrator who is managing the cluster itself and application developers who are deploying pod workloads on a cluster. diff --git a/website/content/en/v0.5.5/tasks/deprovisioning.md b/website/content/en/v0.5.5/tasks/deprovisioning.md new file mode 100644 index 000000000000..b77dbcb3a2df --- /dev/null +++ b/website/content/en/v0.5.5/tasks/deprovisioning.md @@ -0,0 +1,88 @@ +--- +title: "Deprovisioning" +linkTitle: "Deprovisioning" +weight: 10 +--- + +Karpenter sets a Kubernetes [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) on each node it provisions. +The finalizer specifies additional actions the Karpenter controller will take in response to a node deletion request. +These include: + +* Marking the node as unschedulable, so no further pods can be scheduled there. +* Evicting all pods other than daemonsets from the node. +* Terminating the instance from the cloud provider. +* Deleting the node from the Kubernetes cluster. + +## How Karpenter nodes are deprovisioned + +There are both automated and manual ways of deprovisioning nodes provisioned by Karpenter: + +* **Node empty**: Karpenter notes when the last workload (non-daemonset) pod stops running on a node. From that point, Karpenter waits the number of seconds set by `ttlSecondsAfterEmpty` in the provisioner, then Karpenter requests to delete the node. This feature can keep costs down by removing nodes that are no longer being used for workloads. +* **Node expired**: Karpenter requests to delete the node after a set number of seconds, based on the provisioner `ttlSecondsUntilExpired` value, from the time the node was provisioned. One use case for node expiry is to handle node upgrades. Old nodes (with a potentially outdated Kubernetes version or operating system) are deleted, and replaced with nodes on the current version (assuming that you requested the latest version, rather than a specific version). + + {{% alert title="Note" color="primary" %}} + Keep in mind that a small NodeExpiry results in a higher churn in cluster activity. So, for example, if a cluster + brings up all nodes at once, all the pods on those nodes would fall into the same batching window on expiration. + {{% /alert %}} + +* **Node deleted**: You could use `kubectl` to manually remove a single Karpenter node: + + ```bash + # Delete a specific node + kubectl delete node $NODE_NAME + + # Delete all nodes owned any provisioner + kubectl delete nodes -l karpenter.sh/provisioner-name + + # Delete all nodes owned by a specific provisioner + kubectl delete nodes -l karpenter.sh/provisioner-name=$PROVISIONER_NAME + ``` + +Whether through node expiry or manual deletion, Karpenter seeks to follow graceful termination procedures as described in Kubernetes [Graceful node shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdow) documentation. +If the Karpenter controller is removed or fails, the finalizers on the nodes are orphaned and will require manual removal. + + +{{% alert title="Note" color="primary" %}} +By adding the finalizer, Karpenter improves the default Kubernetes process of node deletion. +When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The instance will continue running in EC2, even though there is no longer a node object for it. +The kubelet isn’t watching for its own existence, so if a node is deleted the kubelet doesn’t terminate itself. +All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. +{{% /alert %}} + +## What can cause deprovisioning to fail? + +There are a few cases where requesting to deprovision a Karpenter node will fail. These include Pod Disruption Budgets and pods that have the `do-not-evict` annotation set. + +### Disruption budgets + +Karpenter respects Pod Disruption Budgets (PDBs) by using a backoff retry eviction strategy. Pods will never be forcibly deleted, so pods that fail to shut down will prevent a node from deprovisioning. +Kubernetes PDBs let you specify how much of a Deployment, ReplicationController, ReplicaSet, or StatefulSet must be protected from disruptions when pod eviction requests are made. + +PDBs can be used to strike a balance by protecting the application's availability while still allowing a cluster administrator to manage the cluster. +Here is an example where the pods matching the label `myapp` will block node termination if evicting the pod would reduce the number of available pods below 4. + +```yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: myapp-pdb +spec: + minAvailable: 4 + selector: + matchLabels: + app: myapp +``` + +You can set `minAvailable` or `maxUnavailable` as integers or as a percentage. +Review what [disruptions are](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/), and [how to configure them](https://kubernetes.io/docs/tasks/run-application/configure-pdb/). + +### Pod set to do-not-evict + +If a pod exists with the annotation `karpenter.sh/do-not-evict` on a node, and a request is made to delete the node, Karpenter will not drain any pods from that node or otherwise try to delete the node. +However, if a `do-not-evict` pod is added to a node while the node is draining, the remaining pods will still evict, but that pod will block termination until it is removed. +In either case, the node will be cordoned to prevent additional work from scheduling. + +That annotation is used for pods that you want to run on one node from start to finish without interruption. +Examples might include a real-time, interactive game that you don't want to interrupt or a long batch job (such as you might have with machine learning) that would need to start over if it were interrupted. + +If you want to terminate a `do-not-evict` pod, you can simply remove the annotation and the finalizer will delete the pod and continue the node deprovisioning process. diff --git a/website/content/en/v0.5.5/tasks/provisioning.md b/website/content/en/v0.5.5/tasks/provisioning.md new file mode 100644 index 000000000000..f9db20e043c1 --- /dev/null +++ b/website/content/en/v0.5.5/tasks/provisioning.md @@ -0,0 +1,80 @@ +--- +title: "Provisioning" +linkTitle: "Provisioning" +weight: 5 +--- + +When you first installed Karpenter, you set up a default Provisioner. +The Provisioner sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. +The Provisioner can be set to do things like: + +* Define taints to limit the pods that can run on nodes Karpenter creates +* Limit node creation to certain zones, instance types, and computer architectures +* Set defaults for node expiration + +You can change your provisioner or add other provisioners to Karpenter. +Here are things you should know about Provisioners: + +* Karpenter won't do anything if there is not at least one Provisioner configured. +* Each Provisioner that is configured is looped through by Karpenter. +* If Karpenter encounters a taint in the Provisioner that is not tolerated by a Pod, Karpenter won't use that Provisioner to provision the pod. +* It is recommended to create Provisioners that are mutually exclusive. So no Pod should match multiple Provisioners. If multiple Provisioners are matched, Karpenter will randomly choose which to use. + +If you want to modify or add provisioners to Karpenter, do the following: + +1. Review the following Provisioner documents: + + * [Provisioner](../../getting-started/#provisioner) in the Getting Started guide for a sample default Provisioner + * [Provisioner API](../../provisioner/) for descriptions of Provisioner API values + * [Provisioning Configuration](../../AWS/provisioning) for cloud-specific settings + +2. Apply the new or modified Provisioner to the cluster. + +The following examples illustrate different aspects of Provisioners. +Refer to [Scheduling](../scheduling) to see how the same features are used in Pod specs to determine where pods run. + +## Example: Requirements + +This provisioner limits nodes to specific zones. +It is flexible to both spot and on-demand capacity types. + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: westzones +spec: + requirements: + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b", "us-west-2c"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["spot", "on-demand"] + provider: + instanceProfile: myprofile-cluster101 +``` +With these settings, the provisioner is able to launch nodes in three availability zones and is flexible to both spot and on-demand purchase types. + +## Example: Isolating Expensive Hardware + +A provisioner can be set up to only provision nodes on particular processor types. +The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: gpu +spec: + ttlSecondsAfterEmpty: 60 + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["p3.8xlarge", "p3.16xlarge"] + taints: + - key: nvidia.com/gpu + value: true + effect: “NoSchedule” +``` +In order for a pod to run on a node defined in this provisioner, it must tolerate `nvidia.com/gpu` in its pod spec. diff --git a/website/content/en/v0.5.5/tasks/scheduling.md b/website/content/en/v0.5.5/tasks/scheduling.md new file mode 100755 index 000000000000..8077a99267c1 --- /dev/null +++ b/website/content/en/v0.5.5/tasks/scheduling.md @@ -0,0 +1,300 @@ +--- +title: "Scheduling" +linkTitle: "Scheduling" +weight: 15 +--- + +If your pods have no requirements for how or where to run, you can let Karpenter choose nodes from the full range of available cloud provider resources. +However, by taking advantage of Karpenter's model of layered constraints, you can be sure that the precise type and amount of resources needed are available to your pods. +Reasons for constraining where your pods run could include: + +* Needing to run in zones where dependent applications or storage are available +* Requiring certain kinds of processors or other hardware +* Wanting to use techniques like topology spread to help insure high availability + +Your Cloud Provider defines the first layer of constraints, including all instance types, architectures, zones, and purchase types available to its cloud. +The cluster administrator adds the next layer of constraints by creating one or more provisioners. +The final layer comes from you adding specifications to your Kubernetes pod deployments. +Pod scheduling constraints must fall within a provisioner's constraints or the pods will not deploy. +For example, if the provisioner sets limits that allow only a particular zone to be used, and a pod asks for a different zone, it will not be scheduled. + +Constraints you can request include: + +* **Resource requests**: Request that certain amount of memory or CPU be available. +* **Node selection**: Choose to run on a node that is has a particular label (`nodeSelector`). +* **Node affinity**: Draws a pod to run on nodes with particular attributes (affinity). +* **Topology spread**: Use topology spread to help insure availability of the application. + +Karpenter supports standard Kubernetes scheduling constraints. +This allows you to define a single set of rules that apply to both existing and provisioned capacity. +Pod affinity is a key exception to this rule. + +{{% alert title="Note" color="primary" %}} +Karpenter supports specific [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) that are useful for scheduling. +{{% /alert %}} + +## Resource requests + +Within a Pod spec, you can both make requests and set limits on resources a pod needs, such as CPU and memory. +For example: + +``` +apiVersion: v1 +kind: Pod +metadata: + name: myapp +spec: + containers: + - name: app + image: myimage + resources: + requests: + memory: "128Mi" + cpu: "500m" + limits: + memory: "256Mi" + cpu: "1000m" +``` +In this example, the container is requesting 128MiB of memory and .5 CPU. +Its limits are set to 256MiB of memory and 1 CPU. +Instance type selection math only uses `requests`, but `limits` may be configured to enable resource oversubscription. + + +See [Managing Resources for Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for details on resource types supported by Kubernetes, [Specify a memory request and a memory limit](https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#specify-a-memory-request-and-a-memory-limit) for examples of memory requests, and [Provisioning Configuration](../../aws/provisioning/) for a list of supported resources. + + +## Selecting nodes + +With `nodeSelector` you can ask for a node that matches selected key-value pairs. +This can include well-known labels or custom labels you create yourself. + +While `nodeSelector` is like node affinity, it doesn't have the same "and/or" matchExpressions that affinity has. +So all key-value pairs must match if you use `nodeSelector`. +Also, `nodeSelector` can do only do inclusions, while `affinity` can do inclusions and exclusions (`In` and `NotIn`). + +### Node selectors + +Here is an example of a `nodeSelector` for selecting nodes: + +``` +nodeSelector: + topology.kubernetes.io/zone: us-west-2a + karpenter.sh/capacity-type: spot +``` +This example features a well-known label (`topology.kubernetes.io/zone`) and a label that is well known to Karpenter (`karpenter.sh/capacity-type`). + +If you want to create a custom label, you should do that at the provisioner level. +Then the pod can declare that custom label. + + +See [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) in the Kubernetes documentation for details. + +### Node affinity + +Examples below illustrate how to use Node affinity to include (`In`) and exclude (`NotIn`) objects. +See [Node affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) for details. +When setting rules, the following Node affinity types define how hard or soft each rule is: + +* **requiredDuringSchedulingIgnoredDuringExecution**: This is a hard rule that must be met. +* **preferredDuringSchedulingIgnoredDuringExecution**: This is a preference, but the pod can run on a node where it is not guaranteed. + +The `IgnoredDuringExecution` part of each tells the pod to keep running, even if conditions change on the node so the rules no longer matched. +You can think of these concepts as `required` and `preferred`, since Kubernetes never implemented other variants of these rules. + +All examples below assume that the provisioner doesn't have constraints to prevent those zones from being used. +The first constraint says you could use `us-west-2a` or `us-west-2b`, the second constraint makes it so only `us-west-2b` can be used. + +``` + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2b"] +``` + +Changing the second operator to `NotIn` would allow the pod to run in `us-west-2a` only: + +``` + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" + operator: "NotIn" + values: ["us-west-2b"] +``` + +Continuing to add to the example, `nodeAffinity` lets you define terms so if one term doesn't work it goes to the next one. +Here, if `us-west-2a` is not available, the second term will cause the pod to run on a spot instance in `us-west-2d`. + + +``` + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: # OR + - key: "topology.kubernetes.io/zone" # AND + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" # AND + operator: "NotIn" + values: ["us-west-2b"] + - matchExpressions: # OR + - key: "karpenter.sh/capacity-type" # AND + operator: "In" + values: ["spot"] + - key: "topology.kubernetes.io/zone" # AND + operator: "In" + values: ["us-west-2d"] +``` +In general, Karpenter will go through each of the `nodeSelectorTerms` in order and take the first one that works. +However, if Karpenter fails to provision on the first `nodeSelectorTerms`, it will try again using the second one. +If they all fail, Karpenter will fail to provision the pod. +Karpenter will backoff and retry over time. +So if capacity becomes available, it will schedule the pod without user intervention. + +## Taints and tolerations + +Taints are the opposite of affinity. +Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. +This example shows a Provisioner that was set up with a taint for only running pods that require a GPU, such as the following: + + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: gpu +spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: + - p3.2xlarge + - p3.8xlarge + - p3.16xlarge + taints: + - key: nvidia.com/gpu + value: true + effect: “NoSchedule” +``` + +For a pod to request to run on a node that has provisioner, it could set a toleration as follows: + +``` +apiVersion: v1 +kind: Pod +metadata: + name: mygpupod +spec: + containers: + - name: gpuapp + resources: + requests: + nvidia.com/gpu: 1 + limits: + nvidia.com/gpu: 1 + image: mygpucontainer + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" +``` +See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) in the Kubernetes documentation for details. + +## Topology Spread + +By using the Kubernetes `topologySpreadConstraints` you can ask the provisioner to have pods push away from each other to limit the blast radius of an outage. +Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. +For example: + +``` +spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + dev: jjones + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + dev: jjones + +``` +Adding this to your podspec would result in: + +* Pods being spread across both zones and hosts (`topologyKey`). +* The `dev` `labelSelector` will include all pods with the label of `dev=jjones` in topology calculations. It is recommended to use a selector to match all pods in a deployment. +* No more than one pod difference in the number of pods on each host (`maxSkew`). +For example, if there were three nodes and five pods the pods could be spread 1, 2, 2 or 2, 1, 2 and so on. +If instead the spread were 5, pods could be 5, 0, 0 or 3, 2, 0, or 2, 1, 2 and so on. +* Karpenter is always able to improve skew by launching new nodes in the right zones. Therefore, `whenUnsatisfiable` does not change provisioning behavior. + +See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. + +## Persistent Volume Topology + +Karpenter automatically detects storage scheduling requirements and includes them in node launch decisions. + +In the following example, the `StorageClass` defines zonal topologies for `us-west-2a` and `us-west-2b` and [binding mode `WaitForFirstConsumer`](https://kubernetes.io/docs/concepts/storage/storage-classes/#volume-binding-mode). +When the pod is created, Karpenter follows references from the `Pod` to `PersistentVolumeClaim` to `StorageClass` and identifies that this pod requires storage in `us-west-2a` and `us-west-2b`. +It randomly selects `us-west-2a`, provisions a node in that zone, and binds the pod to the node. +The CSI driver creates a `PersistentVolume` according to the `PersistentVolumeClaim` and gives it a node affinity rule for `us-west-2a`. + +Later on, the pod is deleted and a new pod is created that requests the same claim. This time, Karpenter identifies that a `PersistentVolume` already exists for the `PersistentVolumeClaim`, and includes its zone `us-west-2a` in the pod's scheduling requirements. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: app +spec: + containers: ... + volumes: + - name: storage + persistentVolumeClaim: + claimName: ebs-claim +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: ebs +provisioner: ebs.csi.aws.com +volumeBindingMode: WaitForFirstConsumer +allowedTopologies: +- matchLabelExpressions: + - key: topology.ebs.csi.aws.com/zone + values: ["us-west-2a", "us-west-2b"] +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: ebs-claim +spec: + accessModes: + - ReadWriteOnce + storageClassName: ebs + resources: + requests: + storage: 4Gi +``` + +{{% alert title="Note" color="primary" %}} +☁️ AWS Specific + +The EBS CSI driver uses `topology.ebs.csi.aws.com/zone` instead of the standard `topology.kubernetes.io/zone` label. Karpenter is aware of label aliasing and translates this label into `topology.kubernetes.io/zone` in memory. When configuring a `StorageClass` for the EBS CSI Driver, you must use `topology.ebs.csi.aws.com/zone`. +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} +The topology key `topology.kubernetes.io/region` is not supported. Legacy in-tree CSI providers specify this label. Instead, install an out-of-tree CSI provider. [Learn more about moving to CSI providers.](https://kubernetes.io/blog/2021/12/10/storage-in-tree-to-csi-migration-status-update/#quick-recap-what-is-csi-migration-and-why-migrate) +{{% /alert %}}