From 79e38fbbde6e15070c8554adab092cc92ab6e412 Mon Sep 17 00:00:00 2001 From: Geoffrey Cline Date: Mon, 13 Dec 2021 16:21:07 -0600 Subject: [PATCH] add docs for v0.5.2 and fix nav issues (#960) * v0.5.2 * update nav for new archive dirs * revert makefile changes Co-authored-by: Ellis Tarn --- website/config.toml | 10 +- .../en/v0.4.3-docs/getting-started/_index.md | 3 - website/content/en/v0.5.0-docs/AWS/_index.md | 5 + .../en/v0.5.0-docs/AWS/launch-templates.md | 231 ++++++++++++++ .../en/v0.5.0-docs/AWS/provisioning.md | 144 +++++++++ website/content/en/v0.5.0-docs/_index.md | 27 ++ .../content/en/v0.5.0-docs/concepts/_index.md | 164 ++++++++++ .../en/v0.5.0-docs/development-guide.md | 103 +++++++ .../en/v0.5.0-docs/getting-started/_index.md | 283 ++++++++++++++++++ .../getting-started/cloudformation.yaml | 59 ++++ website/content/en/v0.5.0-docs/provisioner.md | 147 +++++++++ website/content/en/v0.5.0-docs/reinvent.md | 54 ++++ .../content/en/v0.5.0-docs/tasks/_index.md | 7 + .../en/v0.5.0-docs/tasks/deprov-nodes.md | 36 +++ .../en/v0.5.0-docs/tasks/provisioning-task.md | 80 +++++ .../en/v0.5.0-docs/tasks/running-pods.md | 243 +++++++++++++++ website/content/en/v0.5.2-docs/AWS/_index.md | 5 + .../en/v0.5.2-docs/AWS/launch-templates.md | 231 ++++++++++++++ .../en/v0.5.2-docs/AWS/provisioning.md | 144 +++++++++ website/content/en/v0.5.2-docs/_index.md | 27 ++ .../content/en/v0.5.2-docs/concepts/_index.md | 164 ++++++++++ .../en/v0.5.2-docs/development-guide.md | 103 +++++++ .../en/v0.5.2-docs/getting-started/_index.md | 283 ++++++++++++++++++ .../getting-started/cloudformation.yaml | 59 ++++ website/content/en/v0.5.2-docs/provisioner.md | 147 +++++++++ website/content/en/v0.5.2-docs/reinvent.md | 54 ++++ .../content/en/v0.5.2-docs/tasks/_index.md | 7 + .../en/v0.5.2-docs/tasks/deprov-nodes.md | 36 +++ .../en/v0.5.2-docs/tasks/provisioning-task.md | 80 +++++ .../en/v0.5.2-docs/tasks/running-pods.md | 243 +++++++++++++++ 30 files changed, 3173 insertions(+), 6 deletions(-) create mode 100644 website/content/en/v0.5.0-docs/AWS/_index.md create mode 100644 website/content/en/v0.5.0-docs/AWS/launch-templates.md create mode 100644 website/content/en/v0.5.0-docs/AWS/provisioning.md create mode 100755 website/content/en/v0.5.0-docs/_index.md create mode 100644 website/content/en/v0.5.0-docs/concepts/_index.md create mode 100644 website/content/en/v0.5.0-docs/development-guide.md create mode 100644 website/content/en/v0.5.0-docs/getting-started/_index.md create mode 100644 website/content/en/v0.5.0-docs/getting-started/cloudformation.yaml create mode 100644 website/content/en/v0.5.0-docs/provisioner.md create mode 100644 website/content/en/v0.5.0-docs/reinvent.md create mode 100755 website/content/en/v0.5.0-docs/tasks/_index.md create mode 100644 website/content/en/v0.5.0-docs/tasks/deprov-nodes.md create mode 100644 website/content/en/v0.5.0-docs/tasks/provisioning-task.md create mode 100755 website/content/en/v0.5.0-docs/tasks/running-pods.md create mode 100644 website/content/en/v0.5.2-docs/AWS/_index.md create mode 100644 website/content/en/v0.5.2-docs/AWS/launch-templates.md create mode 100644 website/content/en/v0.5.2-docs/AWS/provisioning.md create mode 100755 website/content/en/v0.5.2-docs/_index.md create mode 100644 website/content/en/v0.5.2-docs/concepts/_index.md create mode 100644 website/content/en/v0.5.2-docs/development-guide.md create mode 100644 website/content/en/v0.5.2-docs/getting-started/_index.md create mode 100644 website/content/en/v0.5.2-docs/getting-started/cloudformation.yaml create mode 100644 website/content/en/v0.5.2-docs/provisioner.md create mode 100644 website/content/en/v0.5.2-docs/reinvent.md create mode 100755 website/content/en/v0.5.2-docs/tasks/_index.md create mode 100644 website/content/en/v0.5.2-docs/tasks/deprov-nodes.md create mode 100644 website/content/en/v0.5.2-docs/tasks/provisioning-task.md create mode 100755 website/content/en/v0.5.2-docs/tasks/running-pods.md diff --git a/website/config.toml b/website/config.toml index 65923f4c7edf..4d614b6a057f 100644 --- a/website/config.toml +++ b/website/config.toml @@ -66,7 +66,7 @@ anchor = "smart" ## Site Params [params] -latest_release_version = "0.5.1" +latest_release_version = "0.5.2" copyright = "Amazon.com, Inc. or its affiliates." # Repository configuration (URLs for in-page links to opening issues and suggesting changes) @@ -137,9 +137,13 @@ pre = "" # Add your release versions here [[params.versions]] - version = "v0.5.0" + version = "v0.5.2" url = "/docs/" +[[params.versions]] + version = "v0.5.0" + url = "/v0.5.0-docs/" + [[params.versions]] version = "v0.4.3" - url = "/v0.4.3-docs/" \ No newline at end of file + url = "/v0.4.3-docs/" diff --git a/website/content/en/v0.4.3-docs/getting-started/_index.md b/website/content/en/v0.4.3-docs/getting-started/_index.md index 96d7bff45532..f2f6202d1744 100644 --- a/website/content/en/v0.4.3-docs/getting-started/_index.md +++ b/website/content/en/v0.4.3-docs/getting-started/_index.md @@ -3,9 +3,6 @@ title: "Getting Started with Karpenter on AWS" linkTitle: "Getting Started" weight: 10 -menu: - main: - weight: 10 --- Karpenter automatically provisions new nodes in response to unschedulable diff --git a/website/content/en/v0.5.0-docs/AWS/_index.md b/website/content/en/v0.5.0-docs/AWS/_index.md new file mode 100644 index 000000000000..13f1c3f9e722 --- /dev/null +++ b/website/content/en/v0.5.0-docs/AWS/_index.md @@ -0,0 +1,5 @@ +--- +title: "AWS" +linkTitle: "AWS" +weight: 70 +--- \ No newline at end of file diff --git a/website/content/en/v0.5.0-docs/AWS/launch-templates.md b/website/content/en/v0.5.0-docs/AWS/launch-templates.md new file mode 100644 index 000000000000..1b1e687dd166 --- /dev/null +++ b/website/content/en/v0.5.0-docs/AWS/launch-templates.md @@ -0,0 +1,231 @@ +--- +title: "Launch Templates and Custom Images" +linkTitle: "Launch Templates" +weight: 80 +--- + +By default, Karpenter generates launch templates that use [EKS Optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) for nodes. Often, users need to customize the node image to integrate with existing infrastructure or meet compliance requirements. Karpenter supports custom node images through Launch Templates. If you need to customize the node, then you need a custom launch template. + +Note: By customizing the image, you are taking responsibility for maintaining the image, including security updates. In the default configuration, Karpenter will use the latest version of the EKS optimized AMI, which is maintained by AWS. + + +## Introduction + +Karpenter follows existing AWS patterns for customizing the base image of +instances. More specifically, Karpenter uses [EC2 launch templates](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchTemplates.html). Launch +templates may specify many values. The pivotal value is the base image (AMI). +Launch templates further specify many different parameters related to networking, authorization, instance type, and more. + +Launch Templates and AMIs are unique to AWS regions, similar to EKS clusters. IAM resources are global. + +**Karpenter only implementes a subset of launch template fields, and some fields should not be set.** + +This guide describes requirements for using launch templates with Karpenter, and later an example procedure. + +## Launch Template Requirements + +The Launch Template resource includes a large number of fields. AWS accepts launch templates with any subset of these fields defined. + +Certain fields are obviously critical, such as AMI and User Data. Some fields are useful for particular workloads, such as storage and IAM Instance Profile. + +Finally, **the majority of Launch Template fields should not be set** (or will have no effect), such as network interfaces and instance type. + +## Important Fields + +When creating a custom launch template, the AMI and User Data are the defining characteristics. Instance Profile (IAM Role) and Security Group (firewall rules) are also important for Karpenter. + +### AMI + +AMI (Amazon Machine Image), is the base image/VM for a launch template. + +[Review the instructions for importing a VM to AWS.](https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html) Note the AMI id generated by this process, such as, +`ami-074cce78125f09d61`. + +### User Data - Autoconfigure + +Importantly, the AMI must support automatically connecting to a cluster based +on "user data", or a base64 encoded string passed to the instance at startup. +The syntax and purpose of the user data varies between images. The Karpenter +default OS, Amazon Linux 2 (AL2), accepts shell scripts (bash commands). + +[AWS calls data passed to an instance at launch time "user +data".](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html#user-data-shell-scripts) + +In the default configuration, Karpenter uses an EKS optimized version of AL2 and passes the hostname of the Kubernetes API server, and a certificate. The EKS Optimized AMI includes a `bootstrap.sh` script which connects the instance to the cluster, based on the passed data. + +Alternatively, you may reference AWS's [`bootstrap.sh` +file](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) +when building a custom base image. + +``` +#!/bin/bash +/etc/eks/bootstrap.sh \ +--kubelet-extra-args <'--max-pods=40'> \ +--b64-cluster-ca \ +--apiserver-endpoint \ +--dns-cluster-ip \ +--use-max-pods false +``` + +Note, you must populate this command with live values. Karpenter will +not change the user data in the launch template. + +Encode using yaml function `!Base64` yaml function or `cat userdata.sh | base64 > userdata-encoded.txt` shell command. + +**Bootstrap Script Parameters** + +The sample bootstrap script requires information to join the cluster. + +These values may be found using: +``` +aws eks describe-cluster --name MyKarpenterCluster +``` + +**Kubelet Arguments** + +Specifying max-pods can break Karpenter's binpacking logic (it has no way to know what this setting is). If Karpenter attempts to pack more than this number of pods, the instance may be oversized, and additional pods will reschedule. + +## Situational Fields + +Configure these values in response to a particular use case, such as nodes interacting with another AWS service, or using EBS storage on the node. + +### Instance Profile - IAM + +The launch template must include an "instance profile" -- an IAM role. + +The instance profile must include *at least* the permissions of the default Karpenter node instance profile. See the default role, `KarpenterNodeRole`, in the full example below for more information. + +See also, [the managed policy "AmazonEKSWorkerNodePolicy"](https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonEKSWorkerNodePolicy) which includes permission to describe clusters and subnets. + +### Storage + +Karpenter expects nothing of node storage. Configure as needed for your base +image. + +### Security Groups - Firewall + +The launch template may include a security group (i.e., instance firewall rules) and the security group must be associated with the virtual private cloud (VPC) of the EKS cluster. If none is specified, the default security group of the cluster VPC is used. + +The security group must permit communication with EKS control plane. Outbound access should be permitted for at least: HTTPS on port 443, DNS (UDP and TCP) on port 53, and your subnet's network access control list (network ACL). + +## Fields with Undefined Behavior + +Resources referenced by these fields are controlled by EKS/Karpenter, and not the launch template. + +### Instance Type + +The instance type should not be specified in the launch template. Karpenter +will determine the launch template at run time. + +### Network Interfaces + +The [AWS CNI](https://docs.aws.amazon.com/eks/latest/userguide/pod-networking.html) will configure the network interfaces. Do not configure network instances in the launch template. + +## Creating the Launch Template + +Launch Templates may be created via the web console, the AWS CLI, or +CloudFormation. + +### CloudFormation + +The procedure, in summary, is to: +1. [Create an AMI as described in the EC2 documentation.](https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html) +2. Write a EC2 Launch Template specification including the AMI. +3. Push the specification to AWS with CloudFormation. +4. Update the Provisioner CRD to specify the new Launch Template. + +An example yaml cloudformation definition of a launch template for Karpenter is +provided below. + +CloudFormation yaml is suited for the moderately high configuration density of +launch templates, and creating the unusual InstanceProfile resource. + +You must manually replace these values in the template: +- SecurityGroupID + - list all security groups with `aws ec2 describe-security-groups` +- Parameters in UserData +- AMI + +```yaml +AWSTemplateFormatVersion: '2010-09-09' +Resources: + # create InstanceProfile wrapper on NodeRole + KarpenterNodeInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + InstanceProfileName: "KarpenterNodeInstanceProfile" + Path: "/" + Roles: + - Ref: "KarpenterNodeRole" + # create role with basic permissions for EKS node + KarpenterNodeRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: "KarpenterNodeRole" + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + !Sub "ec2.${AWS::URLSuffix}" + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + MyLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateData: + IamInstanceProfile: + # Get ARN of InstanceProfile defined above + Arn: !GetAtt + - KarpenterNodeInstanceProfile + - Arn + ImageId: ami-074cce78125f09d61 + # UserData is Base64 Encoded + UserData: !Base64 > + #!/bin/bash + /etc/eks/bootstrap.sh 'MyClusterName' \ + --kubelet-extra-args '--node-labels=node.k8s.aws/capacity-type=spot' \ + --b64-cluster-ca 'LS0t....0tCg==' \ + --apiserver-endpoint 'https://B0385BE29EA792E811CB5866D23C856E.gr7.us-east-2.eks.amazonaws.com' + BlockDeviceMappings: + - Ebs: + VolumeSize: 80 + VolumeType: gp3 + DeviceName: /dev/xvda + # The SecurityGroup must be associated with the cluster VPC + SecurityGroupIds: + - sg-a69adfdb + LaunchTemplateName: KarpenterCustomLaunchTemplate +``` + +Create the Launch Template by uploading the CloudFormation yaml file. The +sample yaml creates an IAM Object (InstanceProfile), so `--capabilities +CAPABILITY_NAMED_IAM` must be indicated. + +``` +aws cloudformation create-stack \ + --stack-name KarpenterLaunchTemplateStack \ + --template-body file://$(pwd)/lt-cfn-demo.yaml \ + --capabilities CAPABILITY_NAMED_IAM +``` + +### Define LaunchTemplate for Provisioner + +The LaunchTemplate is ready to be used. Specify it by name in the [Provisioner +CRD](../../provisioner/). Karpenter will use this template when creating new instances. + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +spec: + provider: + launchTemplate: CustomKarpenterLaunchTemplateDemo + +``` diff --git a/website/content/en/v0.5.0-docs/AWS/provisioning.md b/website/content/en/v0.5.0-docs/AWS/provisioning.md new file mode 100644 index 000000000000..fb72201cfd2c --- /dev/null +++ b/website/content/en/v0.5.0-docs/AWS/provisioning.md @@ -0,0 +1,144 @@ +--- +title: "Provisioning Configuration" +linkTitle: "Provisioning" +weight: 10 +--- + +## spec.provider + +This section covers parameters of the AWS Cloud Provider. + +[Review these fields in the code.](https://github.com/awslabs/karpenter/blob/main/pkg/cloudprovider/aws/apis/v1alpha1/provider.go#L33) + +### InstanceProfile +An `InstanceProfile` is a way to pass a single IAM role to an EC2 instance. + +It is required, and specified by name. A suitable `KarpenterNodeRole` is created in the getting started guide. + +``` +spec: + provider: + instanceProfile: MyInstanceProfile +``` + +### LaunchTemplate + +A launch template is a set of configuration values sufficient for launching an EC2 instance (e.g., AMI, storage spec). + +A custom launch template is specified by name. If none is specified, Karpenter will automatically create a launch template. + +Review the [Launch Template documentation](../launch-templates/) to learn how to create a custom one. + +``` +spec: + provider: + launchTemplate: MyLaunchTemplate +``` + +### SubnetSelector + +Karpenter discovers subnets using [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). + +Subnets may be specified by any AWS tag, including `Name`. Selecting tag values using wildcards ("\*") is supported. + +When launching nodes, Karpenter automatically chooses a subnet that matches the desired zone. If multiple subnets exist for a zone, one is chosen randomly. + +**Examples** + +Select all subnets with a specified tag: +``` + subnetSelector: + kubernetes.io/cluster/MyCluster: '*' +``` + +Select subnets by name: +``` + subnetSelector: + Name: subnet-0fcd7006b3754e95e +``` + +Select subnets by an arbitrary AWS tag key/value pair: +``` + subnetSelector: + MySubnetTag: value +``` + +Select subnets using wildcards: +``` + subnetSelector: + Name: *public* + +``` + +### SecurityGroupSelector + +The security group of an instance is comparable to a set of firewall rules. +If no security groups are explicitly listed, Karpenter discovers them using the tag "kubernetes.io/cluster/MyClusterName", similar to subnet discovery. + +EKS creates at least two security groups by default, [review the documentation](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) for more info. + +Security groups may be specified by any AWS tag, including "name". Selecting tags using wildcards ("*") is supported. + +‼️ When launching nodes, Karpenter uses all of the security groups that match the selector. The only exception to this is security groups tagged with the label `kubernets.io/cluster/MyClusterName`. The AWS Load Balancer controller requires that *only a single security group with this tag may be attached to a node*. In this case, Karpenter selects randomly. + +**Examples** + +Select all security groups with a specified tag: +``` +spec: + provider: + securityGroupSelector: + kubernetes.io/cluster/MyKarpenterSecurityGroups: '*' +``` + +Select security groups by name, or another tag: +``` + securityGroupSelector: + Name: sg-01077157b7cf4f5a8 + MySecurityTag: '' # matches all resources with the tag +``` + +Select security groups by name using a wildcard: +``` + subnetSelector: + Name: *public* +``` + +### Tags + +Tags will be added to every EC2 Instance launched by this provisioner. + +``` +spec: + provider: + tags: + InternalAccountingTag: 1234 + dev.corp.net/app: Calculator + dev.corp.net/team: MyTeam +``` + +## Other Resources + +### Accelerators, GPU + +Accelerator (e.g., GPU) values include +- `nvidia.com/gpu` +- `amd.com/gpu` +- `aws.amazon.com/neuron` + +Karpenter supports accelerators, such as GPUs. + + +Additionally, include a resource requirement in the workload manifest. This will cause the GPU dependent pod will be scheduled onto the appropriate node. + +*Accelerator resource in workload manifest (e.g., pod)* + +```yaml +spec: + template: + spec: + containers: + - resources: + limits: + nvidia.com/gpu: "1" +``` diff --git a/website/content/en/v0.5.0-docs/_index.md b/website/content/en/v0.5.0-docs/_index.md new file mode 100755 index 000000000000..a37c93db51e5 --- /dev/null +++ b/website/content/en/v0.5.0-docs/_index.md @@ -0,0 +1,27 @@ + +--- +title: "Documentation" +linkTitle: "Docs" +weight: 20 +cascade: + type: docs +--- +Karpenter is an open-source node provisioning project built for Kubernetes. +Adding Karpenter to a Kubernetes cluster can dramatically improve the efficiency and cost of running workloads on that cluster. +Karpenter is tightly integrated with Kubernetes features to make sure that the right types and amounts of compute resources are available to pods as they are needed. +Karpenter works by: + +* **Watching** for pods that the Kubernetes scheduler has marked as unschedulable +* **Evaluating** scheduling constraints (resource requests, nodeselectors, affinities, tolerations, and topology spread constraints) requested by the pods +* **Provisioning** nodes that meet the requirements of the pods +* **Scheduling** the pods to run on the new nodes +* **Removing** the nodes when the nodes are no longer needed + +As a cluster operator, you can configure an unconstrained Karpenter provisioner when it is first installed and not change it again. +Other times, you might continue to tweak the provisioner or create multiple provisioners for a cluster used by different teams. +On-going cluster operator tasks include upgrading and decomissioning nodes. + +As an application developer, you can make specific requests for capacity and features you want from the nodes running your pods. +Karpenter is designed to quickly create the best possible nodes to meet those needs and schedule the pods to run on them. + +Learn more about Karpenter and how to get started below. diff --git a/website/content/en/v0.5.0-docs/concepts/_index.md b/website/content/en/v0.5.0-docs/concepts/_index.md new file mode 100644 index 000000000000..f029e7371422 --- /dev/null +++ b/website/content/en/v0.5.0-docs/concepts/_index.md @@ -0,0 +1,164 @@ +--- +title: "Concepts" +linkTitle: "Concepts" +weight: 35 +--- + +Users fall under two basic roles: Kubernetes cluster operators and application developers. +This document describes Karpenter concepts through the lens of those two types of users. + +## Cluster operator + +As a Kubernetes cluster operator, you can engage with Karpenter to: + +* Install Karpenter +* Configure provisioners to set constraints and other features for managing nodes +* Deprovision nodes +* Upgrade nodes + +Concepts associated with this role are described below. + + +### Installing Karpenter + +Karpenter is designed to run on a node in your Kubernetes cluster. +As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. + +[Getting Started with Karpenter on AWS](https://karpenter.sh/docs/getting-started/) +describes the process of installing Karpenter on an AWS cloud provider. +Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. +For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. + +Once privileges are in place, Karpenter is deployed with a Helm chart. + +### Configuring provisioners + +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. +To configure Karpenter, you create *provisioners* that define how Karpenter manages unschedulable pods and expires nodes. +Here are some things to know about the Karpenter provisioner: + +* **Unschedulable pods**: Karpenter only attempts to provision pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. + +* **Provisioner CR**: Karpenter defines a Custom Resource called a Provisioner to specify provisioning configuration. +Each provisioner manages a distinct set of nodes, but pods can be scheduled to any provisioner that supports its scheduling constraints. +A provisioner contains constraints that impact the nodes that can be provisioned and attributes of those nodes (such timers for removing nodes). +See [Provisioner API](/docs/provisioner/) for a description of settings and the [Provisioning](../tasks/provisioning-task) task for provisioner examples. + +* **Well-known labels**: The provisioner can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. +See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. +Keep in mind that only a subset of these labels are supported in Karpenter, as described later. + +* **Deprovisioning nodes**: A provisioner can also include time-to-live values to indicate when nodes should be deprovisioned after a set amount of time from when they were created or after they becomes empty of deployed pods. + +* **Multiple provisioners**: Multiple provisioners can be configured on the same cluster. +For example, you might want to configure different teams on the same cluster to run on completely separate capacity. +One team could run on nodes nodes using BottleRocket, while another uses EKSOptimizedAMI. + +Although most use cases are addressed with a single provisioner for multiple teams, multiple provisioners are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different deprovisioning settings. + +### Deprovisioning nodes + +Karpenter deletes nodes when they are no longer needed. + +* **Finalizer**: Karpenter places a finalizer bit on each node it creates. +When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. +Karpenter handles all clean-up work needed to properly delete the node. +* **Node Expiry**: If a node expiry time-to-live value (`ttlSecondsUntilExpired`) is reached, that node is drained of pods and deleted (even if it is still running workloads). +* **Empty nodes**: When the last workload pod running on a Karpenter-managed node is gone, the node is annotated with an emptiness timestamp. +Once that "node empty" time-to-live (`ttlSecondsAfterEmpty`) is reached, finalization is triggered. + +For more details on how Karpenter deletes nodes, see [Deprovisioning nodes](../tasks/deprov-nodes/) for details. + +### Upgrading nodes + +A straight-forward way to upgrade nodes is to set `ttlSecondsUntilExpired`. +Nodes will be terminated after a set period of time and will be replaced with newer nodes. + +Understanding the following concepts will help you in carrying out the tasks just described. + +### Constraints + +The concept of layered constraints is key to using Karpenter. +With no constraints defined in provisioners and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. +Nodes can be created using any instance type and run in any zones. + +An application developer can tighten the constraints defined in a provisioner by the cluster operator by defining additional scheduling constraints in their pod spec. +Refer to the description of Karpenter constraints in the Application Developer section below for details. + +### Scheduling + +Karpenter schedules pods that the Kubernetes scheduler has marked unschedulable. +After solving scheduling constraints and launching capacity, Karpenter optimistically creates the Node object and binds the pod. +This stateless approach helps to avoid race conditions and improves performance. +If something is wrong with the launched node, Kubernetes will automatically migrate the pods to a new node. + +Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. +This is useful if there is additional room in the node due to imperfect packing shape or because workloads finish over time. + +### Cloud provider +Karpenter makes requests to provision new nodes to the associated cloud provider. +The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. +Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. + +While using Kubernetes well-known labels, the provisioner can set some values that are specific to the cloud provider. +So, for example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). + +### Kubernetes cluster autoscaler +Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is +designed to add nodes when requests come in to run pods that cannot be met by current capacity. +Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. +By taking a fresh look at provisioning, Karpenter offers the following improvements: + +* **Designed to handle the full flexibility of the cloud**: +Karpenter has the ability to efficiently address the full range of instance types available through AWS. +Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. + +* **Group-less node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. +This enables it to retry in milliseconds instead of minutes when capacity is unavailable. +It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. + +* **Scheduling enforcement**: Cluster autoscaler doesn’t bind pods to the nodes it creates. +Instead, it relies on the kube-scheduler to make the same scheduling decision after the node has come online. +A node that Karpenter launches has its pods bound immediately. +The kubelet doesn't have to wait for the scheduler or for the node to become ready. +It can start preparing the container runtime immediately, including pre-pulling the image. +This can shave seconds off of node startup latency. + +## Application developer + +As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. +Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. +These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). + +Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. +A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). +The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. + +The Kubernetes scheduler tries to match those constraints with available nodes. +If the pod is unschedulable, Karpenter creates compute resources that match its needs. +When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. + +As long as the requests are not outside of the provisioner's constraints, +Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. +Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. + +So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? + +Kubernetes features that Karpenters supports for scheduling nodes include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). +It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) and [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). + +From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, +you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. +Those that are implemented in Karpenter include: + +* **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 +* **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium +* **topology.kubernetes.io/zone**: For example, topology.kubernetes.io/zone=us-east-1c + +{{% alert title="Note" color="primary" %}} +Don't use `podAffinity` and `podAntiAffinity` to schedule pods on the same or different nodes as other pods. +Kubernetes SIG scalability recommends against these features due to their negative performance impact on the Kubernetes Scheduler (see [KEP 895](https://github.com/kubernetes/enhancements/tree/master/keps/sig-scheduling/895-pod-topology-spread#impact-to-other-features)) and Karpenter doesn't support them for the moment (you can follow their consideration by subscribing to the [issue](https://github.com/aws/karpenter/issues/942)).". +Instead, the Karpenter project recommends `topologySpreadConstraints` to reduce blast radius and `nodeSelectors` and `taints` to implement colocation. +{{% /alert %}} + +For more on how, as a developer, you can add constraints to your pod deployment, see [Running pods](../tasks/running-pods/) for details. diff --git a/website/content/en/v0.5.0-docs/development-guide.md b/website/content/en/v0.5.0-docs/development-guide.md new file mode 100644 index 000000000000..921661900f91 --- /dev/null +++ b/website/content/en/v0.5.0-docs/development-guide.md @@ -0,0 +1,103 @@ +--- +title: "Development Guide" +linkTitle: "Development Guide" +weight: 80 +--- + +## Dependencies + +The following tools are required for contributing to the Karpenter project. + +| Package | Version | Install | +| ------------------------------------------------------------------ | -------- | ---------------------------------------------- | +| [go](https://golang.org/dl/) | v1.15.3+ | [Instructions](https://golang.org/doc/install) | +| [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) | | `brew install kubectl` | +| [helm](https://helm.sh/docs/intro/install/) | | `brew install helm` | +| Other tools | | `make toolchain` | + +## Developing + +### Setup / Teardown + +Based on how you are running your Kubernetes cluster, follow the [Environment specific setup](#environment-specific-setup) to configure your environment before you continue. Once you have your environment set up, to install Karpenter in the Kubernetes cluster specified in your `~/.kube/config` run the following commands. + +``` +CLOUD_PROVIDER= make apply # Install Karpenter +make delete # Uninstall Karpenter +``` + +### Developer Loop +* Make sure dependencies are installed + * Run `make codegen` to make sure yaml manifests are generated + * Run `make toolchain` to install cli tools for building and testing the project +* You will need a personal development image repository (e.g. ECR) + * Make sure you have valid credentials to your development repository. + * `$KO_DOCKER_REPO` must point to your development repository + * Your cluster must have permissions to read from the repository +* If you created your cluster on version 1.19 or above, you may need to tag your subnets as mentioned [here]({{< ref "/docs/getting-started/_index.md#tag-subnets" >}}). This is a temporary problem with our subnet discovery system, and is being tracked [here](https://github.com/aws/karpenter/issues/404#issuecomment-845283904). +* It's also a good idea to persist `$CLOUD_PROVIDER` in your environment variables to simplify the `make apply` command. + +### Build and Deploy +*Note: these commands do not rely on each other and may be executed independently* +```sh +make apply # quickly deploy changes to your cluster +make dev # run codegen, lint, and tests +``` + +### Testing +```sh +make test # E2e correctness tests +make battletest # More rigorous tests run in CI environment +``` + +### Verbose Logging +```sh +kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' +``` + +### Debugging Metrics +OSX: +```sh +open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 +``` + +Linux: +```sh +gio open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 +``` + +### Tailing Logs +While you can tail Karpenter's logs with kubectl, there's a number of tools out there that enhance the experience. We recommend [Stern](https://pkg.go.dev/github.com/planetscale/stern#section-readme): + +```sh +stern -l karpenter=controller -n karpenter +``` + +## Environment specific setup + +### AWS +Set the CLOUD_PROVIDER environment variable to build cloud provider specific packages of Karpenter. + +```sh +export CLOUD_PROVIDER=aws +``` + +For local development on Karpenter you will need a Docker repo which can manage your images for Karpenter components. +You can use the following command to provision an ECR repository. +```sh +aws ecr create-repository \ + --repository-name karpenter/controller \ + --image-scanning-configuration scanOnPush=true \ + --region ${AWS_DEFAULT_REGION} +aws ecr create-repository \ + --repository-name karpenter/webhook \ + --image-scanning-configuration scanOnPush=true \ + --region ${AWS_DEFAULT_REGION} +``` + +Once you have your ECR repository provisioned, configure your Docker daemon to authenticate with your newly created repository. + +```sh +export KO_DOCKER_REPO="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/karpenter" +aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin $KO_DOCKER_REPO +``` diff --git a/website/content/en/v0.5.0-docs/getting-started/_index.md b/website/content/en/v0.5.0-docs/getting-started/_index.md new file mode 100644 index 000000000000..e1f2df9e79e3 --- /dev/null +++ b/website/content/en/v0.5.0-docs/getting-started/_index.md @@ -0,0 +1,283 @@ + +--- +title: "Getting Started with Karpenter on AWS" +linkTitle: "Getting Started" +weight: 10 +--- + +Karpenter automatically provisions new nodes in response to unschedulable +pods. Karpenter does this by observing events within the Kubernetes cluster, +and then sending commands to the underlying cloud provider. + +In this example, the cluster is running on Amazon Web Services (AWS) Elastic +Kubernetes Service (EKS). Karpenter is designed to be cloud provider agnostic, +but currently only supports AWS. Contributions are welcomed. + +This guide should take less than 1 hour to complete, and cost less than $0.25. +Follow the clean-up instructions to reduce any charges. + +## Install + +Karpenter is installed in clusters with a helm chart. + +Karpenter additionally requires IAM Roles for Service Accounts (IRSA). IRSA +permits Karpenter (within the cluster) to make privileged requests to AWS (as +the cloud provider). + +### Required Utilities + +Install these tools before proceeding: + +1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) +2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) +3. `eksctl` - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) + +Login to the AWS CLI with a user that has sufficient privileges to create a +cluster. + +### Environment Variables + +After setting up the tools, set the following environment variables to store +commonly used values. + +```bash +export CLUSTER_NAME=$USER-karpenter-demo +export AWS_DEFAULT_REGION=us-west-2 +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +``` + +### Create a Cluster + +Create a cluster with `eksctl`. This example configuration file specifies a basic cluster with one initial node and sets up an IAM OIDC provider for the cluster to enable IAM roles for pods: + +```bash +cat < cluster.yaml +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: ${CLUSTER_NAME} + region: ${AWS_DEFAULT_REGION} + version: "1.21" +managedNodeGroups: + - instanceType: m5.large + amiFamily: AmazonLinux2 + name: ${CLUSTER_NAME}-ng + desiredCapacity: 1 + minSize: 1 + maxSize: 10 +iam: + withOIDC: true +EOF +eksctl create cluster -f cluster.yaml +``` + +This guide uses a self-managed node group to host Karpenter. + +Karpenter itself can run anywhere, including on [self-managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/worker.html), [managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html), or [AWS Fargate](https://aws.amazon.com/fargate/). + +Karpenter will provision EC2 instances in your account. + +### Tag Subnets + +Karpenter discovers subnets tagged `kubernetes.io/cluster/$CLUSTER_NAME`. Add this tag to subnets associated configured for your cluster. +Retreive the subnet IDs and tag them with the cluster name. + +```bash +SUBNET_IDS=$(aws cloudformation describe-stacks \ + --stack-name eksctl-${CLUSTER_NAME}-cluster \ + --query 'Stacks[].Outputs[?OutputKey==`SubnetsPrivate`].OutputValue' \ + --output text) +aws ec2 create-tags \ + --resources $(echo $SUBNET_IDS | tr ',' '\n') \ + --tags Key="kubernetes.io/cluster/${CLUSTER_NAME}",Value= +``` + +### Create the KarpenterNode IAM Role + +Instances launched by Karpenter must run with an InstanceProfile that grants permissions necessary to run containers and configure networking. Karpenter discovers the InstanceProfile using the name `KarpenterNodeRole-${ClusterName}`. + +First, create the IAM resources using AWS CloudFormation. + +```bash +TEMPOUT=$(mktemp) +curl -fsSL https://karpenter.sh/docs/getting-started/cloudformation.yaml > $TEMPOUT \ +&& aws cloudformation deploy \ + --stack-name Karpenter-${CLUSTER_NAME} \ + --template-file ${TEMPOUT} \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides ClusterName=${CLUSTER_NAME} +``` + +Second, grant access to instances using the profile to connect to the cluster. This command adds the Karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. + +```bash +eksctl create iamidentitymapping \ + --username system:node:{{EC2PrivateDNSName}} \ + --cluster ${CLUSTER_NAME} \ + --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} \ + --group system:bootstrappers \ + --group system:nodes +``` + +Now, Karpenter can launch new EC2 instances and those instances can connect to your cluster. + +### Create the KarpenterController IAM Role + +Karpenter requires permissions like launching instances. This will create an AWS IAM Role, Kubernetes service account, and associate them using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html). + +``` +eksctl create iamserviceaccount \ + --cluster $CLUSTER_NAME --name karpenter --namespace karpenter \ + --attach-policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/KarpenterControllerPolicy-$CLUSTER_NAME \ + --approve +``` + +### Create the EC2 Spot Service Linked Role + +This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). +```bash +aws iam create-service-linked-role --aws-service-name spot.amazonaws.com +# If the role has already been successfully created, you will see: +# An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name AWSServiceRoleForEC2Spot has been taken in this account, please try a different suffix. +``` + +### Install Karpenter Helm Chart + +Use helm to deploy Karpenter to the cluster. + +We created a Kubernetes service account when we created the cluster using +eksctl. Thus, we don't need the helm chart to do that. + +```bash +helm repo add karpenter https://charts.karpenter.sh +helm repo update +helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ + --create-namespace --set serviceAccount.create=false --version {{< param "latest_release_version" >}} \ + --set controller.clusterName=${CLUSTER_NAME} \ + --set controller.clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` + +### Enable Debug Logging (optional) +```sh +kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' +``` + +### Provisioner + +A single Karpenter provisioner is capable of handling many different pod +shapes. Karpenter makes scheduling and provisioning decisions based on pod +attributes such as labels and affinity. In other words, Karpenter eliminates +the need to manage many different node groups. + +Create a default provisioner using the command below. This provisioner +configures instances to connect to your cluster's endpoint and discovers +resources like subnets and security groups using the cluster's name. + +The `ttlSecondsAfterEmpty` value configures Karpenter to terminate empty nodes. +This behavior can be disabled by leaving the value undefined. + +Review the [provisioner CRD](/docs/provisioner/) for more information. For example, +`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. + +Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. + +```bash +cat < + Provisioner API reference page +--- + +## Example Provisioner Resource + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # If nil, the feature is disabled, nodes will never expire + ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; + + # If nil, the feature is disabled, nodes will never scale down due to low utilization + ttlSecondsAfterEmpty: 30 + + # Provisioned nodes will have these taints + # Taints may prevent pods from scheduling if they are not tolerated + taints: + - key: example.com/special-taint + effect: NoSchedule + + # Labels are arbitrary key-values that are applied to all nodes + labels: + billing-team: my-team + + # Requirements that constrain the parameters of provisioned nodes. + # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # Operators { In, NotIn } are supported to enable including or excluding values + requirements: + - key: "node.kubernetes.io/instance-type" + operator: In + values: ["m5.large", "m5.2xlarge"] + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b"] + - key: "kubernetes.io/arch" + operator: In + values: ["arm64", "amd64"] + - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand + operator: In + values: ["spot", "on-demand"] + # These fields vary per cloud provider, see your cloud provider specific documentation + provider: {} +``` + +## spec.requirements + +Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the Provisioner API. + +These well known labels may be specified at the provisioner level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using the both the provisioner's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the provisioner's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. + +For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the provisioner list and the provisioner has instance type requirements, Karpenter will not create a node or schedule the pod. + +📝 None of these values are required. + +### Instance Types + +- key: `node.kubernetes.io/instance-type` + +Generally, instance types should be a list and not a single value. Leaving this field undefined is recommended, as it maximizes choices for efficiently placing pods. + +☁️ **AWS** + +Review [AWS instance types](https://aws.amazon.com/ec2/instance-types/). + +The default value includes all instance types with the exclusion of metal +(non-virtualized), +[non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html), +and GPU instances. + +View the full list of instance types with `aws ec2 describe-instance-types`. + +**Example** + +*Set Default with provisioner.yaml* + +```yaml +spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["m5.large", "m5.2xlarge"] +``` + +*Override with workload manifest (e.g., pod)* + +```yaml +spec: + template: + spec: + nodeSelector: + node.kubernetes.io/instance-type: m5.large +``` + +### Availability Zones + +- key: `topology.kubernetes.io/zone` +- value example: `us-east-1c` + +☁️ **AWS** + +- value list: `aws ec2 describe-availability-zones --region ` + +Karpenter can be configured to create nodes in a particular zone. Note that the Availability Zone `us-east-1a` for your AWS account might not have the same location as `us-east-1a` for another AWS account. + +[Learn more about Availability Zone +IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) + +### Architecture + +- key: `kubernetes.io/arch` +- values + - `amd64` (default) + - `arm64` + +Karpenter supports `amd64` nodes, and `arm64` nodes. + + +### Capacity Type + +- key: `karpenter.sh/capacity-type` + +☁️ **AWS** + +- values + - `spot` (default) + - `on-demand` + +Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). + + +## spec.provider + +This section is cloud provider specific. Reference the appropriate documentation: + +- [AWS](../aws/provisioning/) + + + diff --git a/website/content/en/v0.5.0-docs/reinvent.md b/website/content/en/v0.5.0-docs/reinvent.md new file mode 100644 index 000000000000..45481c63740c --- /dev/null +++ b/website/content/en/v0.5.0-docs/reinvent.md @@ -0,0 +1,54 @@ +# Karpenter re:Invent 2021 Builders Session +​ +![](https://github.com/aws/karpenter/raw/main/website/static/banner.png) +​ +## Prerequisites +Please install the following tools before starting: +- [AWS CLI](https://aws.amazon.com/cli/). If you're on macOS and have [Homebrew](https://brew.sh/) installed, simply `brew install awscli`. Otherwise, follow the AWS CLI [user's guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html). +- [Helm](https://helm.sh/docs/intro/install/), the Kubernetes package manager. If you're on macOS, feel free to simply `brew install helm`. Otherwise, follow the [Helm installation guide](https://helm.sh/docs/intro/install/). +​ +## Get Started +Once you have all the necessary tools installed, configure your shell with the credentials for the temporary AWS account created for this session by: +1. Navigating to the Event Engine team dashboard and clicking on the "☁️ AWS Console" button +2. Configuring your shell with the credentials required by copy and pasting the command for your operating system. +3. Running the following to set your `AWS_ACCOUNT_ID` environmental variable: + ```bash + export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" + ``` +4. Updating your local Kubernetes configuration (`kubeconfig`) by running: + ```bash + aws eks update-kubeconfig --name karpenter-demo --role-arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterEESetupRole-karpenter-demo + ``` +5. Creating an AWS [IAM service-linked role](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html) so that Karpenter can provision Spot EC2 instances with the following command: + ```bash + aws iam create-service-linked-role --aws-service-name spot.amazonaws.com + ``` + _**N.B.** If the role was created previously, you will see:_ + ```bash + # An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name AWSServiceRoleForEC2Spot has been taken in this account, please try a different suffix. + ``` +​ +If you can run the following command and see the pods running in your EKS cluster, you're all set! If not, please ask for help from one of the speakers in the session and they'll get you squared away. For your reference, the cluster name is `karpenter-demo`. +```bash +kubectl get pods -A +``` +​ +Congratulations! You now have access to an Amazon EKS cluster with an EKS Managed Node Group as well as all the AWS infrastructure necessary to use Karpenter. +Happy Building 🔨! +​ +## Install Karpenter + Use the following command to install Karpenter into your cluster: +```bash +helm repo add karpenter https://charts.karpenter.sh +helm repo update +helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ + --create-namespace --set serviceAccount.create=false --version {{< param "latest_release_version" >}} \ + --set controller.clusterName=karpenter-demo \ + --set controller.clusterEndpoint=$(aws eks describe-cluster --name karpenter-demo --query "cluster.endpoint" --output json) \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` +​ +## Next Steps +If you're a Kubernetes expert, feel free to start exploring how Karpenter works on your own and if you have any questions, one of the AWS speakers will be happy to answer them. +​ +If you'd like a guided walkthrough of Karpenter's features and capabilities, you can follow the Karpenter Getting Started guide starting at the ["Provisioner" step](https://karpenter.sh/docs/getting-started/#provisioner). Please don't hesitate to ask your AWS speaker any questions you might have! diff --git a/website/content/en/v0.5.0-docs/tasks/_index.md b/website/content/en/v0.5.0-docs/tasks/_index.md new file mode 100755 index 000000000000..d4da0411c2d6 --- /dev/null +++ b/website/content/en/v0.5.0-docs/tasks/_index.md @@ -0,0 +1,7 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 45 +--- + +Karpenter tasks can be divided into those for a cluster operator who is managing the cluster itself and application developers who are deploying pod workloads on a cluster. diff --git a/website/content/en/v0.5.0-docs/tasks/deprov-nodes.md b/website/content/en/v0.5.0-docs/tasks/deprov-nodes.md new file mode 100644 index 000000000000..59f2cf28bb0d --- /dev/null +++ b/website/content/en/v0.5.0-docs/tasks/deprov-nodes.md @@ -0,0 +1,36 @@ +--- +title: "Deprovisioning nodes" +linkTitle: "Deprovisioning nodes" +weight: 20 +--- + + +## Deletion Workflow + +### Finalizer + +Karpenter adds a finalizer to provisioned nodes. [Review how finalizers work.](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/#how-finalizers-work) + +### Drain Nodes + +Review how to [safely drain a node](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/). + +## Delete Node + +Karpenter changes the behavior of `kubectl delete node`. Nodes will be drained, and then the underlying instance will be deleted. + +## Disruption Budget + +Karpenter respects Pod Disruption Budgets. Review what [disruptions are](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/), and [how to configure them](https://kubernetes.io/docs/tasks/run-application/configure-pdb/). + +Generally, pod workloads may be configured with `.spec.minAvailable` and/or `.spec.maxUnavailable`. Karpenter provisions nodes to accommodate these constraints. + +## Emptiness + +Karpenter will delete nodes (and the instance) that are considered empty of pods. Daemonset pods are not included in this calculation. + +## Expiry + +Nodes may be configured to expire. That is, a maximum lifetime in seconds starting with the node joining the cluster. Review the `ttlSecondsUntilExpired` field of the [provisioner API](../../provisioner/). + +Note that newly created nodes have a Kubernetes version matching the control plane. One use case for node expiry is to handle node upgrades. Old nodes (with a potentially outdated Kubernetes version) are deleted, and replaced with nodes on the current version. diff --git a/website/content/en/v0.5.0-docs/tasks/provisioning-task.md b/website/content/en/v0.5.0-docs/tasks/provisioning-task.md new file mode 100644 index 000000000000..499f332f0c75 --- /dev/null +++ b/website/content/en/v0.5.0-docs/tasks/provisioning-task.md @@ -0,0 +1,80 @@ +--- +title: "Provisioning nodes" +linkTitle: "Provisioning nodes" +weight: 5 +--- + +When you first installed Karpenter, you set up a default Provisioner. +The Provisioner sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. +The Provisioner can be set to do things like: + +* Define taints to limit the pods that can run on nodes Karpenter creates +* Limit node creation to certain zones, instance types, and computer architectures +* Set defaults for node expiration + +You can change your provisioner or add other provisioners to Karpenter. +Here are things you should know about Provisioners: + +* Karpenter won't do anything if there is not at least one Provisioner configured. +* Each Provisioner that is configured is looped through by Karpenter. +* If Karpenter encounters a taint in the Provisioner that is not tolerated by a Pod, Karpenter won't use that Provisioner to provision the pod. +* It is recommended to create Provisioners that are mutually exclusive. So no Pod should match multiple Provisioners. If multiple Provisioners are matched, Karpenter will randomly choose which to use. + +If you want to modify or add provisioners to Karpenter, do the following: + +1. Review the following Provisioner documents: + + * [Provisioner](../../getting-started/#provisioner) in the Getting Started guide for a sample default Provisioner + * [Provisioner API](../../provisioner/) for descriptions of Provisioner API values + * [Provisioning Configuration](../../AWS/provisioning) for cloud-specific settings + +2. Apply the new or modified Provisioner to the cluster. + +The following examples illustrate different aspects of Provisioners. +Refer to [Running pods](../running-pods) to see how the same features are used in Pod specs to determine where pods run. + +## Example: Requirements + +This provisioner limits nodes to specific zones. +It is flexible to both spot and on-demand capacity types. + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: westzones +spec: + requirements: + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b", "us-west-2c"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["spot", "on-demand"] + provider: + instanceProfile: myprofile-cluster101 +``` +With these settings, the provisioner is able to launch nodes in three availability zones and is flexible to both spot and on-demand purchase types. + +## Example: Isolating Expensive Hardware + +A provisioner can be set up to only provision nodes on particular processor types. +The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: gpu +spec: + ttlSecondsAfterEmpty: 60 + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["p3.8xlarge", "p3.16xlarge"] + taints: + - key: nvidia.com/gpu + value: true + effect: “NoSchedule” +``` +In order for a pod to run on a node defined in this provisioner, it must tolerate `nvidia.com/gpu` in its pod spec. diff --git a/website/content/en/v0.5.0-docs/tasks/running-pods.md b/website/content/en/v0.5.0-docs/tasks/running-pods.md new file mode 100755 index 000000000000..1ea1daef3257 --- /dev/null +++ b/website/content/en/v0.5.0-docs/tasks/running-pods.md @@ -0,0 +1,243 @@ +--- +title: "Running pods" +linkTitle: "Running pods" +weight: 10 +--- + +If your pods have no requirements for how or where to run, you can let Karpenter choose nodes from the full range of available cloud provider resources. +However, by taking advantage of Karpenter's model of layered constraints, you can be sure that the precise type and amount of resources needed are available to your pods. +Reasons for constraining where your pods run could include: + +* Needing to run in zones where dependent applications or storage are available +* Requiring certain kinds of processors or other hardware +* Wanting to use techniques like topology spread to help insure high availability + +Your Cloud Provider defines the first layer of constraints, including all instance types, architectures, zones, and purchase types available to its cloud. +The cluster operator adds the next layer of constraints by creating one or more provisioners. +The final layer comes from you adding specifications to your Kubernetes pod deployments. +Pod scheduling constraints must fall within a provisioner's constraints or the pods will not deploy. +For example, if the provisioner sets limits that allow only a particular zone to be used, and a pod asks for a different zone, it will not be scheduled. + +Constraints you can request include: + +* **Resource requests**: Request that certain amount of memory or CPU be available. +* **Node selection**: Choose to run on a node that is has a particular label (`nodeSelector`). +* **Node affinity**: Draws a pod to run on nodes with particular attributes (affinity). +* **Topology spread**: Use topology spread to help insure availability of the application. + +Karpenter supports standard Kubernetes scheduling constraints. +This allows you to define a single set of rules that apply to both existing and provisioned capacity. +Pod affinity is a key exception to this rule. + +{{% alert title="Note" color="primary" %}} +Karpenter supports specific [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) that are useful for scheduling. +{{% /alert %}} + +## Resource requests (`resources`) + +Within a Pod spec, you can both make requests and set limits on resources a pod needs, such as CPU and memory. +For example: + +``` +apiVersion: v1 +kind: Pod +metadata: + name: myapp +spec: + containers: + - name: app + image: myimage + resources: + requests: + memory: "128Mi" + cpu: "500m" + limits: + memory: "256Mi" + cpu: "1000m" +``` +In this example, the container is requesting 128MiB of memory and .5 CPU. +Its limits are set to 256MiB of memory and 1 CPU. +Instance type selection math only uses `requests`, but `limits` may be configured to enable resource oversubscription. + + +See [Managing Resources for Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for details on resource types supported by Kubernetes, [Specify a memory request and a memory limit](https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#specify-a-memory-request-and-a-memory-limit) for examples of memory requests, and [Provisioning Configuration](../../aws/provisioning/) for a list of supported resources. + + +## Selecting nodes (`nodeSelector` and `nodeAffinity`) + +With `nodeSelector` you can ask for a node that matches selected key-value pairs. +This can include well-known labels or custom labels you create yourself. + +While `nodeSelector` is like node affinity, it doesn't have the same "and/or" matchExpressions that affinity has. +So all key-value pairs must match if you use `nodeSelector`. +Also, `nodeSelector` can do only do inclusions, while `affinity` can do inclusions and exclusions (`In` and `NotIn`). + +### Node selector (`nodeSelector`) + +Here is an example of a `nodeSelector` for selecting nodes: + +``` +nodeSelector: + topology.kubernetes.io/zone: us-west-2a + karpenter.sh/capacity-type: spot +``` +This example features a well-known label (`topology.kubernetes.io/zone`) and a label that is well known to Karpenter (`karpenter.sh/capacity-type`). + +If you want to create a custom label, you should do that at the provisioner level. +Then the pod can declare that custom label. + + +See [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) in the Kubernetes documentation for details. + +### Node affinity (`nodeAffinity`) + +Examples below illustrate how to use Node affinity to include (`In`) and exclude (`NotIn`) objects. +See [Node affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) for details. +When setting rules, the following Node affinity types define how hard or soft each rule is: + +* **requiredDuringSchedulingIgnoredDuringExecution**: This is a hard rule that must be met. +* **preferredDuringSchedulingIgnoredDuringExecution**: This is a preference, but the pod can run on a node where it is not guaranteed. + +The `IgnoredDuringExecution` part of each tells the pod to keep running, even if conditions change on the node so the rules no longer matched. +You can think of these concepts as `required` and `preferred`, since Kubernetes never implemented other variants of these rules. + +All examples below assume that the provisioner doesn't have constraints to prevent those zones from being used. +The first constraint says you could use `us-west-2a` or `us-west-2b`, the second constraint makes it so only `us-west-2b` can be used. + +``` + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2b"] +``` + +Changing the second operator to `NotIn` would allow the pod to run in `us-west-2a` only: + +``` + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" + operator: "NotIn" + values: ["us-west-2b"] +``` + +Continuing to add to the example, `nodeAffinity` lets you define terms so if one term doesn't work it goes to the next one. +Here, if `us-west-2a` is not available, the second term will cause the pod to run on a spot instance in `us-west-2d`. + + +``` + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: # OR + - key: "topology.kubernetes.io/zone" # AND + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" # AND + operator: "NotIn" + values: ["us-west-2b"] + - matchExpressions: # OR + - key: "karpenter.sh/capacity-type" # AND + operator: "In" + values: ["spot"] + - key: "topology.kubernetes.io/zone" # AND + operator: "In" + values: ["us-west-2d"] +``` +In general, Karpenter will go through each of the `nodeSelectorTerms` in order and take the first one that works. +However, if Karpenter fails to provision on the first `nodeSelectorTerms`, it will try again using the second one. +If they all fail, Karpenter will fail to provision the pod. +Karpenter will backoff and retry over time. +So if capacity becomes available, it will schedule the pod without user intervention. + +## Taints and tolerations + +Taints are the opposite of affinity. +Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. +This example shows a Provisioner that was set up with a taint for only running pods that require a GPU, such as the following: + + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: gpu +spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: + - p3.2xlarge + - p3.8xlarge + - p3.16xlarge + taints: + - key: nvidia.com/gpu + value: true + effect: “NoSchedule” +``` + +For a pod to request to run on a node that has provisioner, it could set a toleration as follows: + +``` +apiVersion: v1 +kind: Pod +metadata: + name: mygpupod +spec: + containers: + - name: gpuapp + resources: + requests: + nvidia.com/gpu: 1 + limits: + nvidia.com/gpu: 1 + image: mygpucontainer + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" +``` +See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) in the Kubernetes documentation for details. + +## Topology spread (`topologySpreadConstraints`) + +By using the Kubernetes `topologySpreadConstraints` you can ask the provisioner to have pods push away from each other to limit the blast radius of an outage. +Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. +For example: + +``` +spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + dev: jjones + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + dev: jjones + +``` +Adding this to your podspec would result in: + +* Pods being spread across both zones and hosts (`topologyKey`). +* The `dev` `labelSelector` will include all pods with the label of `dev=jjones` in topology calculations. It is recommended to use a selector to match all pods in a deployment. +* No more than one pod difference in the number of pods on each host (`maxSkew`). +For example, if there were three nodes and five pods the pods could be spread 1, 2, 2 or 2, 1, 2 and so on. +If instead the spread were 5, pods could be 5, 0, 0 or 3, 2, 0, or 2, 1, 2 and so on. +* Karpenter is always able to improve skew by launching new nodes in the right zones. Therefore, `whenUnsatisfiable` does not change provisioning behavior. + +See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. diff --git a/website/content/en/v0.5.2-docs/AWS/_index.md b/website/content/en/v0.5.2-docs/AWS/_index.md new file mode 100644 index 000000000000..13f1c3f9e722 --- /dev/null +++ b/website/content/en/v0.5.2-docs/AWS/_index.md @@ -0,0 +1,5 @@ +--- +title: "AWS" +linkTitle: "AWS" +weight: 70 +--- \ No newline at end of file diff --git a/website/content/en/v0.5.2-docs/AWS/launch-templates.md b/website/content/en/v0.5.2-docs/AWS/launch-templates.md new file mode 100644 index 000000000000..1b1e687dd166 --- /dev/null +++ b/website/content/en/v0.5.2-docs/AWS/launch-templates.md @@ -0,0 +1,231 @@ +--- +title: "Launch Templates and Custom Images" +linkTitle: "Launch Templates" +weight: 80 +--- + +By default, Karpenter generates launch templates that use [EKS Optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) for nodes. Often, users need to customize the node image to integrate with existing infrastructure or meet compliance requirements. Karpenter supports custom node images through Launch Templates. If you need to customize the node, then you need a custom launch template. + +Note: By customizing the image, you are taking responsibility for maintaining the image, including security updates. In the default configuration, Karpenter will use the latest version of the EKS optimized AMI, which is maintained by AWS. + + +## Introduction + +Karpenter follows existing AWS patterns for customizing the base image of +instances. More specifically, Karpenter uses [EC2 launch templates](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchTemplates.html). Launch +templates may specify many values. The pivotal value is the base image (AMI). +Launch templates further specify many different parameters related to networking, authorization, instance type, and more. + +Launch Templates and AMIs are unique to AWS regions, similar to EKS clusters. IAM resources are global. + +**Karpenter only implementes a subset of launch template fields, and some fields should not be set.** + +This guide describes requirements for using launch templates with Karpenter, and later an example procedure. + +## Launch Template Requirements + +The Launch Template resource includes a large number of fields. AWS accepts launch templates with any subset of these fields defined. + +Certain fields are obviously critical, such as AMI and User Data. Some fields are useful for particular workloads, such as storage and IAM Instance Profile. + +Finally, **the majority of Launch Template fields should not be set** (or will have no effect), such as network interfaces and instance type. + +## Important Fields + +When creating a custom launch template, the AMI and User Data are the defining characteristics. Instance Profile (IAM Role) and Security Group (firewall rules) are also important for Karpenter. + +### AMI + +AMI (Amazon Machine Image), is the base image/VM for a launch template. + +[Review the instructions for importing a VM to AWS.](https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html) Note the AMI id generated by this process, such as, +`ami-074cce78125f09d61`. + +### User Data - Autoconfigure + +Importantly, the AMI must support automatically connecting to a cluster based +on "user data", or a base64 encoded string passed to the instance at startup. +The syntax and purpose of the user data varies between images. The Karpenter +default OS, Amazon Linux 2 (AL2), accepts shell scripts (bash commands). + +[AWS calls data passed to an instance at launch time "user +data".](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html#user-data-shell-scripts) + +In the default configuration, Karpenter uses an EKS optimized version of AL2 and passes the hostname of the Kubernetes API server, and a certificate. The EKS Optimized AMI includes a `bootstrap.sh` script which connects the instance to the cluster, based on the passed data. + +Alternatively, you may reference AWS's [`bootstrap.sh` +file](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bootstrap.sh) +when building a custom base image. + +``` +#!/bin/bash +/etc/eks/bootstrap.sh \ +--kubelet-extra-args <'--max-pods=40'> \ +--b64-cluster-ca \ +--apiserver-endpoint \ +--dns-cluster-ip \ +--use-max-pods false +``` + +Note, you must populate this command with live values. Karpenter will +not change the user data in the launch template. + +Encode using yaml function `!Base64` yaml function or `cat userdata.sh | base64 > userdata-encoded.txt` shell command. + +**Bootstrap Script Parameters** + +The sample bootstrap script requires information to join the cluster. + +These values may be found using: +``` +aws eks describe-cluster --name MyKarpenterCluster +``` + +**Kubelet Arguments** + +Specifying max-pods can break Karpenter's binpacking logic (it has no way to know what this setting is). If Karpenter attempts to pack more than this number of pods, the instance may be oversized, and additional pods will reschedule. + +## Situational Fields + +Configure these values in response to a particular use case, such as nodes interacting with another AWS service, or using EBS storage on the node. + +### Instance Profile - IAM + +The launch template must include an "instance profile" -- an IAM role. + +The instance profile must include *at least* the permissions of the default Karpenter node instance profile. See the default role, `KarpenterNodeRole`, in the full example below for more information. + +See also, [the managed policy "AmazonEKSWorkerNodePolicy"](https://docs.aws.amazon.com/eks/latest/userguide/security-iam-awsmanpol.html#security-iam-awsmanpol-AmazonEKSWorkerNodePolicy) which includes permission to describe clusters and subnets. + +### Storage + +Karpenter expects nothing of node storage. Configure as needed for your base +image. + +### Security Groups - Firewall + +The launch template may include a security group (i.e., instance firewall rules) and the security group must be associated with the virtual private cloud (VPC) of the EKS cluster. If none is specified, the default security group of the cluster VPC is used. + +The security group must permit communication with EKS control plane. Outbound access should be permitted for at least: HTTPS on port 443, DNS (UDP and TCP) on port 53, and your subnet's network access control list (network ACL). + +## Fields with Undefined Behavior + +Resources referenced by these fields are controlled by EKS/Karpenter, and not the launch template. + +### Instance Type + +The instance type should not be specified in the launch template. Karpenter +will determine the launch template at run time. + +### Network Interfaces + +The [AWS CNI](https://docs.aws.amazon.com/eks/latest/userguide/pod-networking.html) will configure the network interfaces. Do not configure network instances in the launch template. + +## Creating the Launch Template + +Launch Templates may be created via the web console, the AWS CLI, or +CloudFormation. + +### CloudFormation + +The procedure, in summary, is to: +1. [Create an AMI as described in the EC2 documentation.](https://docs.aws.amazon.com/vm-import/latest/userguide/vmimport-image-import.html) +2. Write a EC2 Launch Template specification including the AMI. +3. Push the specification to AWS with CloudFormation. +4. Update the Provisioner CRD to specify the new Launch Template. + +An example yaml cloudformation definition of a launch template for Karpenter is +provided below. + +CloudFormation yaml is suited for the moderately high configuration density of +launch templates, and creating the unusual InstanceProfile resource. + +You must manually replace these values in the template: +- SecurityGroupID + - list all security groups with `aws ec2 describe-security-groups` +- Parameters in UserData +- AMI + +```yaml +AWSTemplateFormatVersion: '2010-09-09' +Resources: + # create InstanceProfile wrapper on NodeRole + KarpenterNodeInstanceProfile: + Type: "AWS::IAM::InstanceProfile" + Properties: + InstanceProfileName: "KarpenterNodeInstanceProfile" + Path: "/" + Roles: + - Ref: "KarpenterNodeRole" + # create role with basic permissions for EKS node + KarpenterNodeRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: "KarpenterNodeRole" + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + !Sub "ec2.${AWS::URLSuffix}" + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" + MyLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateData: + IamInstanceProfile: + # Get ARN of InstanceProfile defined above + Arn: !GetAtt + - KarpenterNodeInstanceProfile + - Arn + ImageId: ami-074cce78125f09d61 + # UserData is Base64 Encoded + UserData: !Base64 > + #!/bin/bash + /etc/eks/bootstrap.sh 'MyClusterName' \ + --kubelet-extra-args '--node-labels=node.k8s.aws/capacity-type=spot' \ + --b64-cluster-ca 'LS0t....0tCg==' \ + --apiserver-endpoint 'https://B0385BE29EA792E811CB5866D23C856E.gr7.us-east-2.eks.amazonaws.com' + BlockDeviceMappings: + - Ebs: + VolumeSize: 80 + VolumeType: gp3 + DeviceName: /dev/xvda + # The SecurityGroup must be associated with the cluster VPC + SecurityGroupIds: + - sg-a69adfdb + LaunchTemplateName: KarpenterCustomLaunchTemplate +``` + +Create the Launch Template by uploading the CloudFormation yaml file. The +sample yaml creates an IAM Object (InstanceProfile), so `--capabilities +CAPABILITY_NAMED_IAM` must be indicated. + +``` +aws cloudformation create-stack \ + --stack-name KarpenterLaunchTemplateStack \ + --template-body file://$(pwd)/lt-cfn-demo.yaml \ + --capabilities CAPABILITY_NAMED_IAM +``` + +### Define LaunchTemplate for Provisioner + +The LaunchTemplate is ready to be used. Specify it by name in the [Provisioner +CRD](../../provisioner/). Karpenter will use this template when creating new instances. + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +spec: + provider: + launchTemplate: CustomKarpenterLaunchTemplateDemo + +``` diff --git a/website/content/en/v0.5.2-docs/AWS/provisioning.md b/website/content/en/v0.5.2-docs/AWS/provisioning.md new file mode 100644 index 000000000000..fb72201cfd2c --- /dev/null +++ b/website/content/en/v0.5.2-docs/AWS/provisioning.md @@ -0,0 +1,144 @@ +--- +title: "Provisioning Configuration" +linkTitle: "Provisioning" +weight: 10 +--- + +## spec.provider + +This section covers parameters of the AWS Cloud Provider. + +[Review these fields in the code.](https://github.com/awslabs/karpenter/blob/main/pkg/cloudprovider/aws/apis/v1alpha1/provider.go#L33) + +### InstanceProfile +An `InstanceProfile` is a way to pass a single IAM role to an EC2 instance. + +It is required, and specified by name. A suitable `KarpenterNodeRole` is created in the getting started guide. + +``` +spec: + provider: + instanceProfile: MyInstanceProfile +``` + +### LaunchTemplate + +A launch template is a set of configuration values sufficient for launching an EC2 instance (e.g., AMI, storage spec). + +A custom launch template is specified by name. If none is specified, Karpenter will automatically create a launch template. + +Review the [Launch Template documentation](../launch-templates/) to learn how to create a custom one. + +``` +spec: + provider: + launchTemplate: MyLaunchTemplate +``` + +### SubnetSelector + +Karpenter discovers subnets using [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). + +Subnets may be specified by any AWS tag, including `Name`. Selecting tag values using wildcards ("\*") is supported. + +When launching nodes, Karpenter automatically chooses a subnet that matches the desired zone. If multiple subnets exist for a zone, one is chosen randomly. + +**Examples** + +Select all subnets with a specified tag: +``` + subnetSelector: + kubernetes.io/cluster/MyCluster: '*' +``` + +Select subnets by name: +``` + subnetSelector: + Name: subnet-0fcd7006b3754e95e +``` + +Select subnets by an arbitrary AWS tag key/value pair: +``` + subnetSelector: + MySubnetTag: value +``` + +Select subnets using wildcards: +``` + subnetSelector: + Name: *public* + +``` + +### SecurityGroupSelector + +The security group of an instance is comparable to a set of firewall rules. +If no security groups are explicitly listed, Karpenter discovers them using the tag "kubernetes.io/cluster/MyClusterName", similar to subnet discovery. + +EKS creates at least two security groups by default, [review the documentation](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) for more info. + +Security groups may be specified by any AWS tag, including "name". Selecting tags using wildcards ("*") is supported. + +‼️ When launching nodes, Karpenter uses all of the security groups that match the selector. The only exception to this is security groups tagged with the label `kubernets.io/cluster/MyClusterName`. The AWS Load Balancer controller requires that *only a single security group with this tag may be attached to a node*. In this case, Karpenter selects randomly. + +**Examples** + +Select all security groups with a specified tag: +``` +spec: + provider: + securityGroupSelector: + kubernetes.io/cluster/MyKarpenterSecurityGroups: '*' +``` + +Select security groups by name, or another tag: +``` + securityGroupSelector: + Name: sg-01077157b7cf4f5a8 + MySecurityTag: '' # matches all resources with the tag +``` + +Select security groups by name using a wildcard: +``` + subnetSelector: + Name: *public* +``` + +### Tags + +Tags will be added to every EC2 Instance launched by this provisioner. + +``` +spec: + provider: + tags: + InternalAccountingTag: 1234 + dev.corp.net/app: Calculator + dev.corp.net/team: MyTeam +``` + +## Other Resources + +### Accelerators, GPU + +Accelerator (e.g., GPU) values include +- `nvidia.com/gpu` +- `amd.com/gpu` +- `aws.amazon.com/neuron` + +Karpenter supports accelerators, such as GPUs. + + +Additionally, include a resource requirement in the workload manifest. This will cause the GPU dependent pod will be scheduled onto the appropriate node. + +*Accelerator resource in workload manifest (e.g., pod)* + +```yaml +spec: + template: + spec: + containers: + - resources: + limits: + nvidia.com/gpu: "1" +``` diff --git a/website/content/en/v0.5.2-docs/_index.md b/website/content/en/v0.5.2-docs/_index.md new file mode 100755 index 000000000000..a37c93db51e5 --- /dev/null +++ b/website/content/en/v0.5.2-docs/_index.md @@ -0,0 +1,27 @@ + +--- +title: "Documentation" +linkTitle: "Docs" +weight: 20 +cascade: + type: docs +--- +Karpenter is an open-source node provisioning project built for Kubernetes. +Adding Karpenter to a Kubernetes cluster can dramatically improve the efficiency and cost of running workloads on that cluster. +Karpenter is tightly integrated with Kubernetes features to make sure that the right types and amounts of compute resources are available to pods as they are needed. +Karpenter works by: + +* **Watching** for pods that the Kubernetes scheduler has marked as unschedulable +* **Evaluating** scheduling constraints (resource requests, nodeselectors, affinities, tolerations, and topology spread constraints) requested by the pods +* **Provisioning** nodes that meet the requirements of the pods +* **Scheduling** the pods to run on the new nodes +* **Removing** the nodes when the nodes are no longer needed + +As a cluster operator, you can configure an unconstrained Karpenter provisioner when it is first installed and not change it again. +Other times, you might continue to tweak the provisioner or create multiple provisioners for a cluster used by different teams. +On-going cluster operator tasks include upgrading and decomissioning nodes. + +As an application developer, you can make specific requests for capacity and features you want from the nodes running your pods. +Karpenter is designed to quickly create the best possible nodes to meet those needs and schedule the pods to run on them. + +Learn more about Karpenter and how to get started below. diff --git a/website/content/en/v0.5.2-docs/concepts/_index.md b/website/content/en/v0.5.2-docs/concepts/_index.md new file mode 100644 index 000000000000..f029e7371422 --- /dev/null +++ b/website/content/en/v0.5.2-docs/concepts/_index.md @@ -0,0 +1,164 @@ +--- +title: "Concepts" +linkTitle: "Concepts" +weight: 35 +--- + +Users fall under two basic roles: Kubernetes cluster operators and application developers. +This document describes Karpenter concepts through the lens of those two types of users. + +## Cluster operator + +As a Kubernetes cluster operator, you can engage with Karpenter to: + +* Install Karpenter +* Configure provisioners to set constraints and other features for managing nodes +* Deprovision nodes +* Upgrade nodes + +Concepts associated with this role are described below. + + +### Installing Karpenter + +Karpenter is designed to run on a node in your Kubernetes cluster. +As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. + +[Getting Started with Karpenter on AWS](https://karpenter.sh/docs/getting-started/) +describes the process of installing Karpenter on an AWS cloud provider. +Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. +For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. + +Once privileges are in place, Karpenter is deployed with a Helm chart. + +### Configuring provisioners + +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. +To configure Karpenter, you create *provisioners* that define how Karpenter manages unschedulable pods and expires nodes. +Here are some things to know about the Karpenter provisioner: + +* **Unschedulable pods**: Karpenter only attempts to provision pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. + +* **Provisioner CR**: Karpenter defines a Custom Resource called a Provisioner to specify provisioning configuration. +Each provisioner manages a distinct set of nodes, but pods can be scheduled to any provisioner that supports its scheduling constraints. +A provisioner contains constraints that impact the nodes that can be provisioned and attributes of those nodes (such timers for removing nodes). +See [Provisioner API](/docs/provisioner/) for a description of settings and the [Provisioning](../tasks/provisioning-task) task for provisioner examples. + +* **Well-known labels**: The provisioner can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. +See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. +Keep in mind that only a subset of these labels are supported in Karpenter, as described later. + +* **Deprovisioning nodes**: A provisioner can also include time-to-live values to indicate when nodes should be deprovisioned after a set amount of time from when they were created or after they becomes empty of deployed pods. + +* **Multiple provisioners**: Multiple provisioners can be configured on the same cluster. +For example, you might want to configure different teams on the same cluster to run on completely separate capacity. +One team could run on nodes nodes using BottleRocket, while another uses EKSOptimizedAMI. + +Although most use cases are addressed with a single provisioner for multiple teams, multiple provisioners are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different deprovisioning settings. + +### Deprovisioning nodes + +Karpenter deletes nodes when they are no longer needed. + +* **Finalizer**: Karpenter places a finalizer bit on each node it creates. +When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. +Karpenter handles all clean-up work needed to properly delete the node. +* **Node Expiry**: If a node expiry time-to-live value (`ttlSecondsUntilExpired`) is reached, that node is drained of pods and deleted (even if it is still running workloads). +* **Empty nodes**: When the last workload pod running on a Karpenter-managed node is gone, the node is annotated with an emptiness timestamp. +Once that "node empty" time-to-live (`ttlSecondsAfterEmpty`) is reached, finalization is triggered. + +For more details on how Karpenter deletes nodes, see [Deprovisioning nodes](../tasks/deprov-nodes/) for details. + +### Upgrading nodes + +A straight-forward way to upgrade nodes is to set `ttlSecondsUntilExpired`. +Nodes will be terminated after a set period of time and will be replaced with newer nodes. + +Understanding the following concepts will help you in carrying out the tasks just described. + +### Constraints + +The concept of layered constraints is key to using Karpenter. +With no constraints defined in provisioners and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. +Nodes can be created using any instance type and run in any zones. + +An application developer can tighten the constraints defined in a provisioner by the cluster operator by defining additional scheduling constraints in their pod spec. +Refer to the description of Karpenter constraints in the Application Developer section below for details. + +### Scheduling + +Karpenter schedules pods that the Kubernetes scheduler has marked unschedulable. +After solving scheduling constraints and launching capacity, Karpenter optimistically creates the Node object and binds the pod. +This stateless approach helps to avoid race conditions and improves performance. +If something is wrong with the launched node, Kubernetes will automatically migrate the pods to a new node. + +Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. +This is useful if there is additional room in the node due to imperfect packing shape or because workloads finish over time. + +### Cloud provider +Karpenter makes requests to provision new nodes to the associated cloud provider. +The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. +Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. + +While using Kubernetes well-known labels, the provisioner can set some values that are specific to the cloud provider. +So, for example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). + +### Kubernetes cluster autoscaler +Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is +designed to add nodes when requests come in to run pods that cannot be met by current capacity. +Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. +By taking a fresh look at provisioning, Karpenter offers the following improvements: + +* **Designed to handle the full flexibility of the cloud**: +Karpenter has the ability to efficiently address the full range of instance types available through AWS. +Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. + +* **Group-less node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. +This enables it to retry in milliseconds instead of minutes when capacity is unavailable. +It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. + +* **Scheduling enforcement**: Cluster autoscaler doesn’t bind pods to the nodes it creates. +Instead, it relies on the kube-scheduler to make the same scheduling decision after the node has come online. +A node that Karpenter launches has its pods bound immediately. +The kubelet doesn't have to wait for the scheduler or for the node to become ready. +It can start preparing the container runtime immediately, including pre-pulling the image. +This can shave seconds off of node startup latency. + +## Application developer + +As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. +Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. +These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). + +Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. +A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). +The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. + +The Kubernetes scheduler tries to match those constraints with available nodes. +If the pod is unschedulable, Karpenter creates compute resources that match its needs. +When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. + +As long as the requests are not outside of the provisioner's constraints, +Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. +Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. + +So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? + +Kubernetes features that Karpenters supports for scheduling nodes include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). +It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) and [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). + +From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, +you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. +Those that are implemented in Karpenter include: + +* **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 +* **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium +* **topology.kubernetes.io/zone**: For example, topology.kubernetes.io/zone=us-east-1c + +{{% alert title="Note" color="primary" %}} +Don't use `podAffinity` and `podAntiAffinity` to schedule pods on the same or different nodes as other pods. +Kubernetes SIG scalability recommends against these features due to their negative performance impact on the Kubernetes Scheduler (see [KEP 895](https://github.com/kubernetes/enhancements/tree/master/keps/sig-scheduling/895-pod-topology-spread#impact-to-other-features)) and Karpenter doesn't support them for the moment (you can follow their consideration by subscribing to the [issue](https://github.com/aws/karpenter/issues/942)).". +Instead, the Karpenter project recommends `topologySpreadConstraints` to reduce blast radius and `nodeSelectors` and `taints` to implement colocation. +{{% /alert %}} + +For more on how, as a developer, you can add constraints to your pod deployment, see [Running pods](../tasks/running-pods/) for details. diff --git a/website/content/en/v0.5.2-docs/development-guide.md b/website/content/en/v0.5.2-docs/development-guide.md new file mode 100644 index 000000000000..921661900f91 --- /dev/null +++ b/website/content/en/v0.5.2-docs/development-guide.md @@ -0,0 +1,103 @@ +--- +title: "Development Guide" +linkTitle: "Development Guide" +weight: 80 +--- + +## Dependencies + +The following tools are required for contributing to the Karpenter project. + +| Package | Version | Install | +| ------------------------------------------------------------------ | -------- | ---------------------------------------------- | +| [go](https://golang.org/dl/) | v1.15.3+ | [Instructions](https://golang.org/doc/install) | +| [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) | | `brew install kubectl` | +| [helm](https://helm.sh/docs/intro/install/) | | `brew install helm` | +| Other tools | | `make toolchain` | + +## Developing + +### Setup / Teardown + +Based on how you are running your Kubernetes cluster, follow the [Environment specific setup](#environment-specific-setup) to configure your environment before you continue. Once you have your environment set up, to install Karpenter in the Kubernetes cluster specified in your `~/.kube/config` run the following commands. + +``` +CLOUD_PROVIDER= make apply # Install Karpenter +make delete # Uninstall Karpenter +``` + +### Developer Loop +* Make sure dependencies are installed + * Run `make codegen` to make sure yaml manifests are generated + * Run `make toolchain` to install cli tools for building and testing the project +* You will need a personal development image repository (e.g. ECR) + * Make sure you have valid credentials to your development repository. + * `$KO_DOCKER_REPO` must point to your development repository + * Your cluster must have permissions to read from the repository +* If you created your cluster on version 1.19 or above, you may need to tag your subnets as mentioned [here]({{< ref "/docs/getting-started/_index.md#tag-subnets" >}}). This is a temporary problem with our subnet discovery system, and is being tracked [here](https://github.com/aws/karpenter/issues/404#issuecomment-845283904). +* It's also a good idea to persist `$CLOUD_PROVIDER` in your environment variables to simplify the `make apply` command. + +### Build and Deploy +*Note: these commands do not rely on each other and may be executed independently* +```sh +make apply # quickly deploy changes to your cluster +make dev # run codegen, lint, and tests +``` + +### Testing +```sh +make test # E2e correctness tests +make battletest # More rigorous tests run in CI environment +``` + +### Verbose Logging +```sh +kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' +``` + +### Debugging Metrics +OSX: +```sh +open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 +``` + +Linux: +```sh +gio open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 +``` + +### Tailing Logs +While you can tail Karpenter's logs with kubectl, there's a number of tools out there that enhance the experience. We recommend [Stern](https://pkg.go.dev/github.com/planetscale/stern#section-readme): + +```sh +stern -l karpenter=controller -n karpenter +``` + +## Environment specific setup + +### AWS +Set the CLOUD_PROVIDER environment variable to build cloud provider specific packages of Karpenter. + +```sh +export CLOUD_PROVIDER=aws +``` + +For local development on Karpenter you will need a Docker repo which can manage your images for Karpenter components. +You can use the following command to provision an ECR repository. +```sh +aws ecr create-repository \ + --repository-name karpenter/controller \ + --image-scanning-configuration scanOnPush=true \ + --region ${AWS_DEFAULT_REGION} +aws ecr create-repository \ + --repository-name karpenter/webhook \ + --image-scanning-configuration scanOnPush=true \ + --region ${AWS_DEFAULT_REGION} +``` + +Once you have your ECR repository provisioned, configure your Docker daemon to authenticate with your newly created repository. + +```sh +export KO_DOCKER_REPO="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/karpenter" +aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin $KO_DOCKER_REPO +``` diff --git a/website/content/en/v0.5.2-docs/getting-started/_index.md b/website/content/en/v0.5.2-docs/getting-started/_index.md new file mode 100644 index 000000000000..e1f2df9e79e3 --- /dev/null +++ b/website/content/en/v0.5.2-docs/getting-started/_index.md @@ -0,0 +1,283 @@ + +--- +title: "Getting Started with Karpenter on AWS" +linkTitle: "Getting Started" +weight: 10 +--- + +Karpenter automatically provisions new nodes in response to unschedulable +pods. Karpenter does this by observing events within the Kubernetes cluster, +and then sending commands to the underlying cloud provider. + +In this example, the cluster is running on Amazon Web Services (AWS) Elastic +Kubernetes Service (EKS). Karpenter is designed to be cloud provider agnostic, +but currently only supports AWS. Contributions are welcomed. + +This guide should take less than 1 hour to complete, and cost less than $0.25. +Follow the clean-up instructions to reduce any charges. + +## Install + +Karpenter is installed in clusters with a helm chart. + +Karpenter additionally requires IAM Roles for Service Accounts (IRSA). IRSA +permits Karpenter (within the cluster) to make privileged requests to AWS (as +the cloud provider). + +### Required Utilities + +Install these tools before proceeding: + +1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) +2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) +3. `eksctl` - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) + +Login to the AWS CLI with a user that has sufficient privileges to create a +cluster. + +### Environment Variables + +After setting up the tools, set the following environment variables to store +commonly used values. + +```bash +export CLUSTER_NAME=$USER-karpenter-demo +export AWS_DEFAULT_REGION=us-west-2 +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +``` + +### Create a Cluster + +Create a cluster with `eksctl`. This example configuration file specifies a basic cluster with one initial node and sets up an IAM OIDC provider for the cluster to enable IAM roles for pods: + +```bash +cat < cluster.yaml +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: ${CLUSTER_NAME} + region: ${AWS_DEFAULT_REGION} + version: "1.21" +managedNodeGroups: + - instanceType: m5.large + amiFamily: AmazonLinux2 + name: ${CLUSTER_NAME}-ng + desiredCapacity: 1 + minSize: 1 + maxSize: 10 +iam: + withOIDC: true +EOF +eksctl create cluster -f cluster.yaml +``` + +This guide uses a self-managed node group to host Karpenter. + +Karpenter itself can run anywhere, including on [self-managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/worker.html), [managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html), or [AWS Fargate](https://aws.amazon.com/fargate/). + +Karpenter will provision EC2 instances in your account. + +### Tag Subnets + +Karpenter discovers subnets tagged `kubernetes.io/cluster/$CLUSTER_NAME`. Add this tag to subnets associated configured for your cluster. +Retreive the subnet IDs and tag them with the cluster name. + +```bash +SUBNET_IDS=$(aws cloudformation describe-stacks \ + --stack-name eksctl-${CLUSTER_NAME}-cluster \ + --query 'Stacks[].Outputs[?OutputKey==`SubnetsPrivate`].OutputValue' \ + --output text) +aws ec2 create-tags \ + --resources $(echo $SUBNET_IDS | tr ',' '\n') \ + --tags Key="kubernetes.io/cluster/${CLUSTER_NAME}",Value= +``` + +### Create the KarpenterNode IAM Role + +Instances launched by Karpenter must run with an InstanceProfile that grants permissions necessary to run containers and configure networking. Karpenter discovers the InstanceProfile using the name `KarpenterNodeRole-${ClusterName}`. + +First, create the IAM resources using AWS CloudFormation. + +```bash +TEMPOUT=$(mktemp) +curl -fsSL https://karpenter.sh/docs/getting-started/cloudformation.yaml > $TEMPOUT \ +&& aws cloudformation deploy \ + --stack-name Karpenter-${CLUSTER_NAME} \ + --template-file ${TEMPOUT} \ + --capabilities CAPABILITY_NAMED_IAM \ + --parameter-overrides ClusterName=${CLUSTER_NAME} +``` + +Second, grant access to instances using the profile to connect to the cluster. This command adds the Karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. + +```bash +eksctl create iamidentitymapping \ + --username system:node:{{EC2PrivateDNSName}} \ + --cluster ${CLUSTER_NAME} \ + --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} \ + --group system:bootstrappers \ + --group system:nodes +``` + +Now, Karpenter can launch new EC2 instances and those instances can connect to your cluster. + +### Create the KarpenterController IAM Role + +Karpenter requires permissions like launching instances. This will create an AWS IAM Role, Kubernetes service account, and associate them using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html). + +``` +eksctl create iamserviceaccount \ + --cluster $CLUSTER_NAME --name karpenter --namespace karpenter \ + --attach-policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/KarpenterControllerPolicy-$CLUSTER_NAME \ + --approve +``` + +### Create the EC2 Spot Service Linked Role + +This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). +```bash +aws iam create-service-linked-role --aws-service-name spot.amazonaws.com +# If the role has already been successfully created, you will see: +# An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name AWSServiceRoleForEC2Spot has been taken in this account, please try a different suffix. +``` + +### Install Karpenter Helm Chart + +Use helm to deploy Karpenter to the cluster. + +We created a Kubernetes service account when we created the cluster using +eksctl. Thus, we don't need the helm chart to do that. + +```bash +helm repo add karpenter https://charts.karpenter.sh +helm repo update +helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ + --create-namespace --set serviceAccount.create=false --version {{< param "latest_release_version" >}} \ + --set controller.clusterName=${CLUSTER_NAME} \ + --set controller.clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` + +### Enable Debug Logging (optional) +```sh +kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' +``` + +### Provisioner + +A single Karpenter provisioner is capable of handling many different pod +shapes. Karpenter makes scheduling and provisioning decisions based on pod +attributes such as labels and affinity. In other words, Karpenter eliminates +the need to manage many different node groups. + +Create a default provisioner using the command below. This provisioner +configures instances to connect to your cluster's endpoint and discovers +resources like subnets and security groups using the cluster's name. + +The `ttlSecondsAfterEmpty` value configures Karpenter to terminate empty nodes. +This behavior can be disabled by leaving the value undefined. + +Review the [provisioner CRD](/docs/provisioner/) for more information. For example, +`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. + +Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. + +```bash +cat < + Provisioner API reference page +--- + +## Example Provisioner Resource + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + # If nil, the feature is disabled, nodes will never expire + ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; + + # If nil, the feature is disabled, nodes will never scale down due to low utilization + ttlSecondsAfterEmpty: 30 + + # Provisioned nodes will have these taints + # Taints may prevent pods from scheduling if they are not tolerated + taints: + - key: example.com/special-taint + effect: NoSchedule + + # Labels are arbitrary key-values that are applied to all nodes + labels: + billing-team: my-team + + # Requirements that constrain the parameters of provisioned nodes. + # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # Operators { In, NotIn } are supported to enable including or excluding values + requirements: + - key: "node.kubernetes.io/instance-type" + operator: In + values: ["m5.large", "m5.2xlarge"] + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b"] + - key: "kubernetes.io/arch" + operator: In + values: ["arm64", "amd64"] + - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand + operator: In + values: ["spot", "on-demand"] + # These fields vary per cloud provider, see your cloud provider specific documentation + provider: {} +``` + +## spec.requirements + +Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the Provisioner API. + +These well known labels may be specified at the provisioner level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using the both the provisioner's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the provisioner's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. + +For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the provisioner list and the provisioner has instance type requirements, Karpenter will not create a node or schedule the pod. + +📝 None of these values are required. + +### Instance Types + +- key: `node.kubernetes.io/instance-type` + +Generally, instance types should be a list and not a single value. Leaving this field undefined is recommended, as it maximizes choices for efficiently placing pods. + +☁️ **AWS** + +Review [AWS instance types](https://aws.amazon.com/ec2/instance-types/). + +The default value includes all instance types with the exclusion of metal +(non-virtualized), +[non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html), +and GPU instances. + +View the full list of instance types with `aws ec2 describe-instance-types`. + +**Example** + +*Set Default with provisioner.yaml* + +```yaml +spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["m5.large", "m5.2xlarge"] +``` + +*Override with workload manifest (e.g., pod)* + +```yaml +spec: + template: + spec: + nodeSelector: + node.kubernetes.io/instance-type: m5.large +``` + +### Availability Zones + +- key: `topology.kubernetes.io/zone` +- value example: `us-east-1c` + +☁️ **AWS** + +- value list: `aws ec2 describe-availability-zones --region ` + +Karpenter can be configured to create nodes in a particular zone. Note that the Availability Zone `us-east-1a` for your AWS account might not have the same location as `us-east-1a` for another AWS account. + +[Learn more about Availability Zone +IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) + +### Architecture + +- key: `kubernetes.io/arch` +- values + - `amd64` (default) + - `arm64` + +Karpenter supports `amd64` nodes, and `arm64` nodes. + + +### Capacity Type + +- key: `karpenter.sh/capacity-type` + +☁️ **AWS** + +- values + - `spot` (default) + - `on-demand` + +Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). + + +## spec.provider + +This section is cloud provider specific. Reference the appropriate documentation: + +- [AWS](../aws/provisioning/) + + + diff --git a/website/content/en/v0.5.2-docs/reinvent.md b/website/content/en/v0.5.2-docs/reinvent.md new file mode 100644 index 000000000000..45481c63740c --- /dev/null +++ b/website/content/en/v0.5.2-docs/reinvent.md @@ -0,0 +1,54 @@ +# Karpenter re:Invent 2021 Builders Session +​ +![](https://github.com/aws/karpenter/raw/main/website/static/banner.png) +​ +## Prerequisites +Please install the following tools before starting: +- [AWS CLI](https://aws.amazon.com/cli/). If you're on macOS and have [Homebrew](https://brew.sh/) installed, simply `brew install awscli`. Otherwise, follow the AWS CLI [user's guide](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html). +- [Helm](https://helm.sh/docs/intro/install/), the Kubernetes package manager. If you're on macOS, feel free to simply `brew install helm`. Otherwise, follow the [Helm installation guide](https://helm.sh/docs/intro/install/). +​ +## Get Started +Once you have all the necessary tools installed, configure your shell with the credentials for the temporary AWS account created for this session by: +1. Navigating to the Event Engine team dashboard and clicking on the "☁️ AWS Console" button +2. Configuring your shell with the credentials required by copy and pasting the command for your operating system. +3. Running the following to set your `AWS_ACCOUNT_ID` environmental variable: + ```bash + export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" + ``` +4. Updating your local Kubernetes configuration (`kubeconfig`) by running: + ```bash + aws eks update-kubeconfig --name karpenter-demo --role-arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterEESetupRole-karpenter-demo + ``` +5. Creating an AWS [IAM service-linked role](https://docs.aws.amazon.com/IAM/latest/UserGuide/using-service-linked-roles.html) so that Karpenter can provision Spot EC2 instances with the following command: + ```bash + aws iam create-service-linked-role --aws-service-name spot.amazonaws.com + ``` + _**N.B.** If the role was created previously, you will see:_ + ```bash + # An error occurred (InvalidInput) when calling the CreateServiceLinkedRole operation: Service role name AWSServiceRoleForEC2Spot has been taken in this account, please try a different suffix. + ``` +​ +If you can run the following command and see the pods running in your EKS cluster, you're all set! If not, please ask for help from one of the speakers in the session and they'll get you squared away. For your reference, the cluster name is `karpenter-demo`. +```bash +kubectl get pods -A +``` +​ +Congratulations! You now have access to an Amazon EKS cluster with an EKS Managed Node Group as well as all the AWS infrastructure necessary to use Karpenter. +Happy Building 🔨! +​ +## Install Karpenter + Use the following command to install Karpenter into your cluster: +```bash +helm repo add karpenter https://charts.karpenter.sh +helm repo update +helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ + --create-namespace --set serviceAccount.create=false --version {{< param "latest_release_version" >}} \ + --set controller.clusterName=karpenter-demo \ + --set controller.clusterEndpoint=$(aws eks describe-cluster --name karpenter-demo --query "cluster.endpoint" --output json) \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` +​ +## Next Steps +If you're a Kubernetes expert, feel free to start exploring how Karpenter works on your own and if you have any questions, one of the AWS speakers will be happy to answer them. +​ +If you'd like a guided walkthrough of Karpenter's features and capabilities, you can follow the Karpenter Getting Started guide starting at the ["Provisioner" step](https://karpenter.sh/docs/getting-started/#provisioner). Please don't hesitate to ask your AWS speaker any questions you might have! diff --git a/website/content/en/v0.5.2-docs/tasks/_index.md b/website/content/en/v0.5.2-docs/tasks/_index.md new file mode 100755 index 000000000000..d4da0411c2d6 --- /dev/null +++ b/website/content/en/v0.5.2-docs/tasks/_index.md @@ -0,0 +1,7 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 45 +--- + +Karpenter tasks can be divided into those for a cluster operator who is managing the cluster itself and application developers who are deploying pod workloads on a cluster. diff --git a/website/content/en/v0.5.2-docs/tasks/deprov-nodes.md b/website/content/en/v0.5.2-docs/tasks/deprov-nodes.md new file mode 100644 index 000000000000..59f2cf28bb0d --- /dev/null +++ b/website/content/en/v0.5.2-docs/tasks/deprov-nodes.md @@ -0,0 +1,36 @@ +--- +title: "Deprovisioning nodes" +linkTitle: "Deprovisioning nodes" +weight: 20 +--- + + +## Deletion Workflow + +### Finalizer + +Karpenter adds a finalizer to provisioned nodes. [Review how finalizers work.](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/#how-finalizers-work) + +### Drain Nodes + +Review how to [safely drain a node](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/). + +## Delete Node + +Karpenter changes the behavior of `kubectl delete node`. Nodes will be drained, and then the underlying instance will be deleted. + +## Disruption Budget + +Karpenter respects Pod Disruption Budgets. Review what [disruptions are](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/), and [how to configure them](https://kubernetes.io/docs/tasks/run-application/configure-pdb/). + +Generally, pod workloads may be configured with `.spec.minAvailable` and/or `.spec.maxUnavailable`. Karpenter provisions nodes to accommodate these constraints. + +## Emptiness + +Karpenter will delete nodes (and the instance) that are considered empty of pods. Daemonset pods are not included in this calculation. + +## Expiry + +Nodes may be configured to expire. That is, a maximum lifetime in seconds starting with the node joining the cluster. Review the `ttlSecondsUntilExpired` field of the [provisioner API](../../provisioner/). + +Note that newly created nodes have a Kubernetes version matching the control plane. One use case for node expiry is to handle node upgrades. Old nodes (with a potentially outdated Kubernetes version) are deleted, and replaced with nodes on the current version. diff --git a/website/content/en/v0.5.2-docs/tasks/provisioning-task.md b/website/content/en/v0.5.2-docs/tasks/provisioning-task.md new file mode 100644 index 000000000000..499f332f0c75 --- /dev/null +++ b/website/content/en/v0.5.2-docs/tasks/provisioning-task.md @@ -0,0 +1,80 @@ +--- +title: "Provisioning nodes" +linkTitle: "Provisioning nodes" +weight: 5 +--- + +When you first installed Karpenter, you set up a default Provisioner. +The Provisioner sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. +The Provisioner can be set to do things like: + +* Define taints to limit the pods that can run on nodes Karpenter creates +* Limit node creation to certain zones, instance types, and computer architectures +* Set defaults for node expiration + +You can change your provisioner or add other provisioners to Karpenter. +Here are things you should know about Provisioners: + +* Karpenter won't do anything if there is not at least one Provisioner configured. +* Each Provisioner that is configured is looped through by Karpenter. +* If Karpenter encounters a taint in the Provisioner that is not tolerated by a Pod, Karpenter won't use that Provisioner to provision the pod. +* It is recommended to create Provisioners that are mutually exclusive. So no Pod should match multiple Provisioners. If multiple Provisioners are matched, Karpenter will randomly choose which to use. + +If you want to modify or add provisioners to Karpenter, do the following: + +1. Review the following Provisioner documents: + + * [Provisioner](../../getting-started/#provisioner) in the Getting Started guide for a sample default Provisioner + * [Provisioner API](../../provisioner/) for descriptions of Provisioner API values + * [Provisioning Configuration](../../AWS/provisioning) for cloud-specific settings + +2. Apply the new or modified Provisioner to the cluster. + +The following examples illustrate different aspects of Provisioners. +Refer to [Running pods](../running-pods) to see how the same features are used in Pod specs to determine where pods run. + +## Example: Requirements + +This provisioner limits nodes to specific zones. +It is flexible to both spot and on-demand capacity types. + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: westzones +spec: + requirements: + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b", "us-west-2c"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["spot", "on-demand"] + provider: + instanceProfile: myprofile-cluster101 +``` +With these settings, the provisioner is able to launch nodes in three availability zones and is flexible to both spot and on-demand purchase types. + +## Example: Isolating Expensive Hardware + +A provisioner can be set up to only provision nodes on particular processor types. +The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: gpu +spec: + ttlSecondsAfterEmpty: 60 + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["p3.8xlarge", "p3.16xlarge"] + taints: + - key: nvidia.com/gpu + value: true + effect: “NoSchedule” +``` +In order for a pod to run on a node defined in this provisioner, it must tolerate `nvidia.com/gpu` in its pod spec. diff --git a/website/content/en/v0.5.2-docs/tasks/running-pods.md b/website/content/en/v0.5.2-docs/tasks/running-pods.md new file mode 100755 index 000000000000..1ea1daef3257 --- /dev/null +++ b/website/content/en/v0.5.2-docs/tasks/running-pods.md @@ -0,0 +1,243 @@ +--- +title: "Running pods" +linkTitle: "Running pods" +weight: 10 +--- + +If your pods have no requirements for how or where to run, you can let Karpenter choose nodes from the full range of available cloud provider resources. +However, by taking advantage of Karpenter's model of layered constraints, you can be sure that the precise type and amount of resources needed are available to your pods. +Reasons for constraining where your pods run could include: + +* Needing to run in zones where dependent applications or storage are available +* Requiring certain kinds of processors or other hardware +* Wanting to use techniques like topology spread to help insure high availability + +Your Cloud Provider defines the first layer of constraints, including all instance types, architectures, zones, and purchase types available to its cloud. +The cluster operator adds the next layer of constraints by creating one or more provisioners. +The final layer comes from you adding specifications to your Kubernetes pod deployments. +Pod scheduling constraints must fall within a provisioner's constraints or the pods will not deploy. +For example, if the provisioner sets limits that allow only a particular zone to be used, and a pod asks for a different zone, it will not be scheduled. + +Constraints you can request include: + +* **Resource requests**: Request that certain amount of memory or CPU be available. +* **Node selection**: Choose to run on a node that is has a particular label (`nodeSelector`). +* **Node affinity**: Draws a pod to run on nodes with particular attributes (affinity). +* **Topology spread**: Use topology spread to help insure availability of the application. + +Karpenter supports standard Kubernetes scheduling constraints. +This allows you to define a single set of rules that apply to both existing and provisioned capacity. +Pod affinity is a key exception to this rule. + +{{% alert title="Note" color="primary" %}} +Karpenter supports specific [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) that are useful for scheduling. +{{% /alert %}} + +## Resource requests (`resources`) + +Within a Pod spec, you can both make requests and set limits on resources a pod needs, such as CPU and memory. +For example: + +``` +apiVersion: v1 +kind: Pod +metadata: + name: myapp +spec: + containers: + - name: app + image: myimage + resources: + requests: + memory: "128Mi" + cpu: "500m" + limits: + memory: "256Mi" + cpu: "1000m" +``` +In this example, the container is requesting 128MiB of memory and .5 CPU. +Its limits are set to 256MiB of memory and 1 CPU. +Instance type selection math only uses `requests`, but `limits` may be configured to enable resource oversubscription. + + +See [Managing Resources for Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for details on resource types supported by Kubernetes, [Specify a memory request and a memory limit](https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#specify-a-memory-request-and-a-memory-limit) for examples of memory requests, and [Provisioning Configuration](../../aws/provisioning/) for a list of supported resources. + + +## Selecting nodes (`nodeSelector` and `nodeAffinity`) + +With `nodeSelector` you can ask for a node that matches selected key-value pairs. +This can include well-known labels or custom labels you create yourself. + +While `nodeSelector` is like node affinity, it doesn't have the same "and/or" matchExpressions that affinity has. +So all key-value pairs must match if you use `nodeSelector`. +Also, `nodeSelector` can do only do inclusions, while `affinity` can do inclusions and exclusions (`In` and `NotIn`). + +### Node selector (`nodeSelector`) + +Here is an example of a `nodeSelector` for selecting nodes: + +``` +nodeSelector: + topology.kubernetes.io/zone: us-west-2a + karpenter.sh/capacity-type: spot +``` +This example features a well-known label (`topology.kubernetes.io/zone`) and a label that is well known to Karpenter (`karpenter.sh/capacity-type`). + +If you want to create a custom label, you should do that at the provisioner level. +Then the pod can declare that custom label. + + +See [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector) in the Kubernetes documentation for details. + +### Node affinity (`nodeAffinity`) + +Examples below illustrate how to use Node affinity to include (`In`) and exclude (`NotIn`) objects. +See [Node affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) for details. +When setting rules, the following Node affinity types define how hard or soft each rule is: + +* **requiredDuringSchedulingIgnoredDuringExecution**: This is a hard rule that must be met. +* **preferredDuringSchedulingIgnoredDuringExecution**: This is a preference, but the pod can run on a node where it is not guaranteed. + +The `IgnoredDuringExecution` part of each tells the pod to keep running, even if conditions change on the node so the rules no longer matched. +You can think of these concepts as `required` and `preferred`, since Kubernetes never implemented other variants of these rules. + +All examples below assume that the provisioner doesn't have constraints to prevent those zones from being used. +The first constraint says you could use `us-west-2a` or `us-west-2b`, the second constraint makes it so only `us-west-2b` can be used. + +``` + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2b"] +``` + +Changing the second operator to `NotIn` would allow the pod to run in `us-west-2a` only: + +``` + - key: "topology.kubernetes.io/zone" + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" + operator: "NotIn" + values: ["us-west-2b"] +``` + +Continuing to add to the example, `nodeAffinity` lets you define terms so if one term doesn't work it goes to the next one. +Here, if `us-west-2a` is not available, the second term will cause the pod to run on a spot instance in `us-west-2d`. + + +``` + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: # OR + - key: "topology.kubernetes.io/zone" # AND + operator: "In" + values: ["us-west-2a, us-west-2b"] + - key: "topology.kubernetes.io/zone" # AND + operator: "NotIn" + values: ["us-west-2b"] + - matchExpressions: # OR + - key: "karpenter.sh/capacity-type" # AND + operator: "In" + values: ["spot"] + - key: "topology.kubernetes.io/zone" # AND + operator: "In" + values: ["us-west-2d"] +``` +In general, Karpenter will go through each of the `nodeSelectorTerms` in order and take the first one that works. +However, if Karpenter fails to provision on the first `nodeSelectorTerms`, it will try again using the second one. +If they all fail, Karpenter will fail to provision the pod. +Karpenter will backoff and retry over time. +So if capacity becomes available, it will schedule the pod without user intervention. + +## Taints and tolerations + +Taints are the opposite of affinity. +Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. +This example shows a Provisioner that was set up with a taint for only running pods that require a GPU, such as the following: + + +``` +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: gpu +spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: + - p3.2xlarge + - p3.8xlarge + - p3.16xlarge + taints: + - key: nvidia.com/gpu + value: true + effect: “NoSchedule” +``` + +For a pod to request to run on a node that has provisioner, it could set a toleration as follows: + +``` +apiVersion: v1 +kind: Pod +metadata: + name: mygpupod +spec: + containers: + - name: gpuapp + resources: + requests: + nvidia.com/gpu: 1 + limits: + nvidia.com/gpu: 1 + image: mygpucontainer + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" +``` +See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) in the Kubernetes documentation for details. + +## Topology spread (`topologySpreadConstraints`) + +By using the Kubernetes `topologySpreadConstraints` you can ask the provisioner to have pods push away from each other to limit the blast radius of an outage. +Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. +For example: + +``` +spec: + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + dev: jjones + - maxSkew: 1 + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + dev: jjones + +``` +Adding this to your podspec would result in: + +* Pods being spread across both zones and hosts (`topologyKey`). +* The `dev` `labelSelector` will include all pods with the label of `dev=jjones` in topology calculations. It is recommended to use a selector to match all pods in a deployment. +* No more than one pod difference in the number of pods on each host (`maxSkew`). +For example, if there were three nodes and five pods the pods could be spread 1, 2, 2 or 2, 1, 2 and so on. +If instead the spread were 5, pods could be 5, 0, 0 or 3, 2, 0, or 2, 1, 2 and so on. +* Karpenter is always able to improve skew by launching new nodes in the right zones. Therefore, `whenUnsatisfiable` does not change provisioning behavior. + +See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details.