From 150ab7bbcad40b9bce8c8e8a1d41bf15ad45b1c0 Mon Sep 17 00:00:00 2001 From: Carlos Manzanedo Rueda Date: Fri, 1 Jul 2022 05:38:19 +0100 Subject: [PATCH 1/2] upgrade of the workshop to 0.13.1 --- .../040_karpenter/advanced_provisioner.md | 83 ++++++++++++++----- .../automatic_node_provisioning.md | 59 ++++++------- .../040_karpenter/ec2_spot_deployments.md | 2 +- .../040_karpenter/multiple_architectures.md | 49 ++++++----- .../040_karpenter/set_up_the_environment.md | 4 +- .../040_karpenter/set_up_the_provisioner.md | 37 ++++++--- .../using_alternative_provisioners.md | 31 ++++--- 7 files changed, 164 insertions(+), 101 deletions(-) diff --git a/content/karpenter/040_karpenter/advanced_provisioner.md b/content/karpenter/040_karpenter/advanced_provisioner.md index b25514aa..0b9ef7b5 100644 --- a/content/karpenter/040_karpenter/advanced_provisioner.md +++ b/content/karpenter/040_karpenter/advanced_provisioner.md @@ -39,15 +39,26 @@ spec: values: ["amd64","arm64"] limits: resources: - cpu: 256 - provider: - subnetSelector: - alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} - securityGroupSelector: - alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} - tags: - accountingEC2Tag: KarpenterDevEnvironmentEC2 + cpu: 1000 + memory: 1000Gi ttlSecondsAfterEmpty: 30 + ttlSecondsUntilExpired: 2592000 + providerRef: + name: default +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + subnetSelector: + alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + securityGroupSelector: + alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + tags: + KarpenerProvisionerName: "default" + NodeType: "karpenter-workshop" + IntentLabel: "apps" EOF ``` @@ -71,18 +82,35 @@ spec: values: ["amd64","arm64"] limits: resources: - cpu: 128 - provider: - subnetSelector: - alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} - securityGroupSelector: - alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} - tags: - accountingEC2Tag: KarpenterDevEnvironmentEC2 + cpu: 1000 + memory: 1000Gi ttlSecondsAfterEmpty: 30 + ttlSecondsUntilExpired: 2592000 taints: - effect: NoSchedule key: team1 + providerRef: + name: team1 +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: team1 +spec: + amiFamily: Bottlerocket + subnetSelector: + alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + securityGroupSelector: + alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + tags: + KarpenerProvisionerName: "team1" + NodeType: "karpenter-workshop" + IntentLabel: "apps" + userData: | + [settings.kubernetes] + kube-api-qps = 30 + [settings.kubernetes.eviction-hard] + "memory.available" = "20%" EOF ``` @@ -96,6 +124,8 @@ Let's spend some time covering a few points in the Provisioners configuration. * The `team1` Provisioner does only support Pods or Jobs that provide a Toleration for the key `team1`. Nodes procured by this provisioner will be tainted using the Taint with key `team1` and effect `NoSchedule`. +* The `team1` Provisioner does define a different `AWSNodeTemplate` and changes the AMI from the default [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) to [bottlerocket](https://aws.amazon.com/bottlerocket/). It does also adapts the UserData bootstrapping for this particular provider. + {{% notice note %}} If Karpenter encounters a taint in the Provisioner that is not tolerated by a Pod, Karpenter won’t use that Provisioner to provision the pod. It is recommended to create Provisioners that are mutually exclusive. So no Pod should match multiple Provisioners. If multiple Provisioners are matched, Karpenter will randomly choose which to use. {{% /notice %}} @@ -113,13 +143,21 @@ The command should list both the `default` and the `team1` provisioner. We can a kubectl describe provisioners default ``` +You can repeat the same commands with `kubectl get AWSNodeTemplate` to check the provider section within the provisioner. + ## (Optional Read) Customizing AMIs and Node Bootstrapping {{% notice info %}} -In this workshop we will stick to the default AMI's used by Karpenter. This section does not contain any exercise or command. The section describes how the AMI and node bootsrapping can be adapted when needed. If you want to deep dive into this topic you can [read the following karpenter documentation link](https://karpenter.sh/v0.10.0/aws/launch-templates/) +In this workshop we will stick to the default AMI's used by Karpenter. This section does not contain any exercise or command. The section describes how the AMI and node bootsrapping can be adapted when needed. If you want to deep dive into this topic you can [read the following karpenter documentation link](https://karpenter.sh/v0.13.1/aws/launch-templates/) {{% /notice %}} -By default, Karpenter generates [launch templates](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchTemplates.html) that use [EKS Optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) and Encrypted EBS root volumes with the default (AWS Managed) KMS key for nodes. Often, users need to customize the node image to integrate with existing infrastructure, meet compliance requirements, add extra storage, etc. Karpenter supports custom node images and bootsrapping through Launch Templates. If you need to customize the node, then you need a custom launch template. +By default, Karpenter generates [launch templates](https://docs.aws.amazon.com/autoscaling/ec2/userguide/LaunchTemplates.html) that use [EKS Optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) and Encrypted EBS root volumes with the default (AWS Managed) KMS key for nodes. Karpenter does also support [Bottlerocket](https://aws.amazon.com/bottlerocket/). + +Under the hood Karpenter manages the life-cycle the [LaunchTemplates](https://docs.aws.amazon.com/autoscaling/ec2/userguide/launch-templates.html) that point to the right AMI's. Karpenter does also dynamically update new versions of the EKS and Bottlerocket AMI's when there are new available. To automate the Patching of AMI's, Provisioners can use `ttlSecondsUntilExpired` to force nodes to terminate so that new nodes that use the new AMI's replace older versions. + +Karpenter supports passing a `UserData` configuration for EKS (AL2, and Ubuntu as well) and BottleRocket AMIs. This significantly simplifies the management of nodes while enabling users to customize their nodes for aspects such as governance, security or just optimizing with specific bootstrapping parameter. You can read more about how bootstrapping can be applied **[here](https://karpenter.sh/v0.13.1/aws/user-data/)**. Note Bottlerocket and EKS have different data formats, MIME for EKS,AL2 and Ubuntu and TOML for Bottlerocket. + +Often, users may need to customize the AMI to integrate with existing infrastructure, meet compliance requirements, add extra storage, etc. Karpenter supports custom node images and bootsrapping through Launch Templates. If you need to customize the node, then you need a custom launch template. {{%notice note %}} Using custom launch templates prevents multi-architecture support, the ability to automatically upgrade nodes, and securityGroup discovery. Using launch templates may also cause confusion because certain fields are duplicated within Karpenter’s provisioners while others are ignored by Karpenter, e.g. subnets and instance types. @@ -134,11 +172,12 @@ By customizing the image, you are taking responsibility for maintaining it, incl The selection of the Launch Template can be configured in the provider by setting up the `launchTemplate` property. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: team1 spec: - provider: - launchTemplate: CustomKarpenterLaunchTemplateDemo + launchTemplate: CustomKarpenterLaunchTemplateDemo ... ``` diff --git a/content/karpenter/040_karpenter/automatic_node_provisioning.md b/content/karpenter/040_karpenter/automatic_node_provisioning.md index 7ca60c5b..08ee5262 100644 --- a/content/karpenter/040_karpenter/automatic_node_provisioning.md +++ b/content/karpenter/040_karpenter/automatic_node_provisioning.md @@ -96,31 +96,31 @@ echo type: $(kubectl describe node --selector=intent=apps | grep "beta.kubernete There is something even more interesting to learn about how the node was provisioned. Check out Karpenter logs and look at the new Karpenter created. The lines should be similar to the ones below ```bash -2022-05-12T03:38:15.698Z INFO controller Batched 1 pod(s) in 1.000075807s {"commit": "00661aa"} -2022-05-12T03:38:16.485Z DEBUG controller Discovered 401 EC2 instance types {"commit": "00661aa"} -2022-05-12T03:38:16.601Z DEBUG controller Discovered EC2 instance types zonal offerings {"commit": "00661aa"} -2022-05-12T03:38:16.768Z DEBUG controller Discovered subnets: [subnet-0204b1b3b885ca98d (eu-west-1a) subnet-037d1d97a6a473fd1 (eu-west-1b) subnet-04c2ca248972479e7 (eu-west-1b) subnet-063d5c7ba912986d5 (eu-west-1a)] {"commit": "00661aa"} -2022-05-12T03:38:16.953Z DEBUG controller Discovered security groups: [sg-03ab1d5d49b00b596 sg-06e7e2ca961ab3bed] {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:38:16.955Z DEBUG controller Discovered kubernetes version 1.21 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:38:17.046Z DEBUG controller Discovered ami-0440c10a3f77514d8 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:38:17.213Z DEBUG controller Created launch template, Karpenter-eksworkshop-eksctl-7600085100718942941 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:38:19.400Z INFO controller Launched instance: i-0f47f9dc3fa486c35, hostname: ip-192-168-37-165.eu-west-1.compute.internal, type: t3.medium, zone: eu-west-1b, capacityType: spot {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:38:19.412Z INFO controller Created node with 1 pods requesting {"cpu":"1125m","memory":"1536Mi","pods":"3"} from types c4.large, c6a.large, t3a.medium, c5a.large, c6i.large and 306 other(s) {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:38:19.426Z INFO controller Waiting for unschedulable pods {"commit": "00661aa"} +2022-07-01T03:00:19.634Z INFO controller.provisioning Found 1 provisionable pod(s) {"commit": "1f7a67b"} +2022-07-01T03:00:19.634Z INFO controller.provisioning Computed 1 new node(s) will fit 1 pod(s) {"commit": "1f7a67b"} +2022-07-01T03:00:19.790Z DEBUG controller.provisioning.cloudprovider Discovered subnets: [subnet-0e528fbbaf13542c2 (eu-west-1b) subnet-0a9bd9b668d8ae58d (eu-west-1a) subnet-03aec03eee186dc42 (eu-west-1a) subnet-03ff683f2535bcd8d (eu-west-1b)] {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:00:19.871Z DEBUG controller.provisioning.cloudprovider Discovered security groups: [sg-076f0ca74b68addb2 sg-09176f21ae53f5d60] {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:00:19.873Z DEBUG controller.provisioning.cloudprovider Discovered kubernetes version 1.21 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:00:19.928Z DEBUG controller.provisioning.cloudprovider Discovered ami-0413b176c68479e84 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:00:19.972Z DEBUG controller.provisioning.cloudprovider Discovered launch template Karpenter-eksworkshop-eksctl-12663282710833670681 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:00:23.013Z INFO controller.provisioning.cloudprovider Launched instance: i-05e8535378b1caf35, hostname: ip-192-168-36-234.eu-west-1.compute.internal, type: c5a.xlarge, zone: eu-west-1b, capacityType: spot {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:00:23.042Z INFO controller.provisioning Created node with 1 pods requesting {"cpu":"1125m","memory":"1536Mi","pods":"3"} from types t3a.xlarge, c6a.xlarge, c5a.xlarge, t3.xlarge, c6i.xlarge and 333 other(s) {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:00:23.042Z INFO controller.provisioning Waiting for unschedulable pods {"commit": "1f7a67b"} +2022-07-01T03:00:23.042Z DEBUG controller.events Normal {"commit": "1f7a67b", "object": {"kind":"Pod","namespace":"default","name":"inflate-b9d769f59-rcjnj","uid":"e0f98d1d-eaf6-46ff-9ea0-4d66a6842815","apiVersion":"v1","resourceVersion":"20925"}, "reason": "NominatePod", "message": "Pod should schedule on ip-192-168-36-234.eu-west-1.compute.internal"} ``` We explained earlier on about group-less cluster scalers and how that simplifies operations and maintenance. Let's deep dive for a second into this concept. Notice how Karpenter picks up the instance from a diversified selection of instances. In this case it selected the following instances: ``` -c4.large, c6a.large, t3a.medium, c5a.large, c6i.large and 306 other(s) +from types t3a.xlarge, c6a.xlarge, c5a.xlarge, t3.xlarge, c6i.xlarge and 333 other(s) ``` -{{% notice note %}} -Instances types might be different depending on the region selected. -{{% /notice %}} +**Note** how the types, 'nano', 'micro', 'small', 'medium', 'large', where filtered for this selection. While our recommendation is to diversify on as many instances as possible, there are cases where provisioners may want to filter smaller (or specific) instances types. +Instances types might be different depending on the region selected. + All this instances are the suitable instances that reduce the waste of resources (memory and CPU) for the pod submitted. If you are interested in Algorithms, internally Karpenter is using a [First Fit Decreasing (FFD)](https://en.wikipedia.org/wiki/Bin_packing_problem#First_Fit_Decreasing_(FFD)) approach. Note however this can change in the future. We did set Karpenter Provisioner to use [EC2 Spot instances](https://aws.amazon.com/ec2/spot/), and there was no `instance-types` [requirement section in the Provisioner to filter the type of instances](https://karpenter.sh/v0.10.0/provisioner/#instance-types). This means that Karpenter will use the default value of instances types to use. The default value includes all instance types with the exclusion of metal (non-virtualized), [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html), and GPU instances.Internally Karpenter used **EC2 Fleet in Instant mode** to provision the instances. You can read more about EC2 Fleet Instant mode [**here**](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instant-fleet.html). Here are a few properties to mention about EC2 Fleet instant mode that are key for Karpenter. @@ -215,38 +215,39 @@ Both Karpenter and [Cluster Autoscaler](https://github.com/kubernetes/autoscaler {{% /expand %}} -#### 6) How would you scale the number of replicas to 6? What do you expect to happen? Which instance types were selected in this case ? +#### 6) How would you scale the number of replicas to 10? What do you expect to happen? Which instance types were selected in this case ? {{%expand "Click here to show the answer" %}} This one should be easy! ``` -kubectl scale deployment inflate --replicas 6 +kubectl scale deployment inflate --replicas 10 ``` This will set a few pods pending. Karpenter will get the pending pod signal and run a new provisioning cycle similar to the one below (confirm by checking Karpenter logs). This time, the capacity should get provisioned with a slightly different set of characteristics. Given the new size of aggregated pod requirements, Karpenter will check which type of instance diversification makes sense to use. ```bash -2022-05-12T03:47:17.907Z INFO controller Batched 5 pod(s) in 1.056343494s {"commit": "00661aa"} -2022-05-12T03:47:18.692Z DEBUG controller Discovered 401 EC2 instance types {"commit": "00661aa"} -2022-05-12T03:47:18.848Z DEBUG controller Discovered EC2 instance types zonal offerings {"commit": "00661aa"} -2022-05-12T03:47:19.011Z DEBUG controller Discovered subnets: [subnet-0204b1b3b885ca98d (eu-west-1a) subnet-037d1d97a6a473fd1 (eu-west-1b) subnet-04c2ca248972479e7 (eu-west-1b) subnet-063d5c7ba912986d5 (eu-west-1a)] {"commit": "00661aa"}2022-05-12T03:47:19.094Z DEBUG controller Discovered security groups: [sg-03ab1d5d49b00b596 sg-06e7e2ca961ab3bed] {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:47:19.097Z DEBUG controller Discovered kubernetes version 1.21 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:47:19.134Z DEBUG controller Discovered ami-0440c10a3f77514d8 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:47:19.175Z DEBUG controller Discovered launch template Karpenter-eksworkshop-eksctl-7600085100718942941 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:47:21.199Z INFO controller Launched instance: i-066971cf53a56a2f7, hostname: ip-192-168-38-49.eu-west-1.compute.internal, type: t3.2xlarge, zone: eu-west-1b, capacityType: spot {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:47:21.208Z INFO controller Created node with 5 pods requesting {"cpu":"5125m","memory":"7680Mi","pods":"7"} from types c4.2xlarge, c6i.2xlarge, c6a.2xlarge, c5a.2xlarge, c5.2xlarge and 222 other(s) {"commit": "00661aa", "provisioner": "default"} -2022-05-12T03:47:21.236Z INFO controller Waiting for unschedulable pods {"commit": "00661aa"} +2022-07-01T03:13:32.754Z INFO controller.provisioning Found 7 provisionable pod(s) {"commit": "1f7a67b"} +2022-07-01T03:13:32.754Z INFO controller.provisioning Computed 1 new node(s) will fit 7 pod(s) {"commit": "1f7a67b"} +2022-07-01T03:13:32.824Z DEBUG controller.provisioning.cloudprovider Discovered subnets: [subnet-0e528fbbaf13542c2 (eu-west-1b) subnet-0a9bd9b668d8ae58d (eu-west-1a) subnet-03aec03eee186dc42 (eu-west-1a) subnet-03ff683f2535bcd8d (eu-west-1b)] {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:13:32.867Z DEBUG controller.provisioning.cloudprovider Discovered security groups: [sg-076f0ca74b68addb2 sg-09176f21ae53f5d60] {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:13:32.868Z DEBUG controller.provisioning.cloudprovider Discovered kubernetes version 1.21 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:13:32.929Z DEBUG controller.provisioning.cloudprovider Discovered ami-0413b176c68479e84 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:13:33.105Z DEBUG controller.provisioning.cloudprovider Created launch template, Karpenter-eksworkshop-eksctl-12663282710833670681 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:13:35.047Z INFO controller.provisioning.cloudprovider Launched instance: i-004d9de653118ae9d, hostname: ip-192-168-27-254.eu-west-1.compute.internal, type: t3a.2xlarge, zone: eu-west-1a, capacityType: spot {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:13:35.074Z INFO controller.provisioning Created node with 7 pods requesting {"cpu":"7125m","memory":"10752Mi","pods":"9"} from types t3a.2xlarge, c6a.2xlarge, c5a.2xlarge, t3.2xlarge, c6i.2xlarge and 276 other(s) {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T03:13:35.074Z INFO controller.provisioning Waiting for unschedulable pods {"commit": "1f7a67b"} ``` Indeed the instances selected this time are larger ! The instances selected in this example were: ```bash -c4.2xlarge, c6i.2xlarge, c6a.2xlarge, c5a.2xlarge, c5.2xlarge and 222 other(s) +from types t3a.2xlarge, c6a.2xlarge, c5a.2xlarge, t3.2xlarge, c6i.2xlarge and 276 other(s) ``` + Finally to check out the configuration of the `intent=apps` node execute again: ``` @@ -280,5 +281,5 @@ In this section we have learned: * Karpenter uses cordon and drain [best practices](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/) to terminate nodes. The configuration of when a node is terminated can be controlled with `ttlSecondsAfterEmpty` -* Karpenter can scale up from zero and scale in to zero. +* Karpenter can scale-out from zero when applications have available working pods and scale-in to zero when there are no running jobs or pods. diff --git a/content/karpenter/040_karpenter/ec2_spot_deployments.md b/content/karpenter/040_karpenter/ec2_spot_deployments.md index e34575a4..1f9c7a33 100644 --- a/content/karpenter/040_karpenter/ec2_spot_deployments.md +++ b/content/karpenter/040_karpenter/ec2_spot_deployments.md @@ -24,7 +24,7 @@ To deploy the Node Termination Handler run the following command: helm repo add eks https://aws.github.io/eks-charts helm install aws-node-termination-handler \ --namespace kube-system \ - --version 0.18.3 \ + --version 0.18.5 \ --set nodeSelector."karpenter\\.sh/capacity-type"=spot \ eks/aws-node-termination-handler ``` diff --git a/content/karpenter/040_karpenter/multiple_architectures.md b/content/karpenter/040_karpenter/multiple_architectures.md index cc04d245..df35ac8c 100644 --- a/content/karpenter/040_karpenter/multiple_architectures.md +++ b/content/karpenter/040_karpenter/multiple_architectures.md @@ -42,6 +42,7 @@ spec: nodeSelector: intent: apps kubernetes.io/arch: amd64 + karpenter.sh/capacity-type: on-demand containers: - image: public.ecr.aws/eks-distro/kubernetes/pause:3.2 name: inflate-amd64 @@ -124,21 +125,24 @@ The output should show something similar to the lines below ```bash ... -2022-05-12T04:15:10.593Z INFO controller Batched 2 pod(s) in 1.009818195s {"commit": "00661aa"} -2022-05-12T04:15:10.770Z DEBUG controller Discovered subnets: [subnet-0204b1b3b885ca98d (eu-west-1a) subnet-037d1d97a6a473fd1 (eu-west-1b) subnet-04c2ca248972479e7 (eu-west-1b) subnet-063d5c7ba912986d5 (eu-west-1a)] {"commit": "00661aa"} -2022-05-12T04:15:10.851Z DEBUG controller Discovered security groups: [sg-03ab1d5d49b00b596 sg-06e7e2ca961ab3bed]{"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:15:10.854Z DEBUG controller Discovered kubernetes version 1.21 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:15:10.940Z DEBUG controller Discovered ami-0440c10a3f77514d8 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:15:10.977Z DEBUG controller Discovered launch template Karpenter-eksworkshop-eksctl-13001267661656074018 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:15:13.081Z INFO controller Launched instance: i-0d19d3eeb2d59578d, hostname: ip-192-168-22-165.eu-west-1.compute.internal, type: m4.xlarge, zone: eu-west-1a, capacityType: spot {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:15:13.093Z INFO controller Created node with 2 pods requesting {"cpu":"2125m","memory":"512M","pods":"5"} from types c4.xlarge, c6i.xlarge, c5a.xlarge, c5.xlarge, c6a.xlarge and 267 other(s) {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:15:13.103Z INFO controller Waiting for unschedulable pods {"commit": "00661aa"} +2022-07-01T04:02:12.087Z DEBUG controller.provisioning Discovered 531 EC2 instance types {"commit": "1f7a67b"} +2022-07-01T04:02:12.246Z DEBUG controller.provisioning Discovered subnets: [subnet-0e528fbbaf13542c2 (eu-west-1b) subnet-0a9bd9b668d8ae58d (eu-west-1a) subnet-03aec03eee186dc42 (eu-west-1a) subnet-03ff683f2535bcd8d (eu-west-1b)] {"commit": "1f7a67b"} +2022-07-01T04:02:12.397Z DEBUG controller.provisioning Discovered EC2 instance types zonal offerings {"commit": "1f7a67b"} +2022-07-01T04:02:12.598Z INFO controller.provisioning Found 2 provisionable pod(s) {"commit": "1f7a67b"} +2022-07-01T04:02:12.598Z INFO controller.provisioning Computed 1 new node(s) will fit 2 pod(s) {"commit": "1f7a67b"} +2022-07-01T04:02:12.690Z DEBUG controller.provisioning.cloudprovider Discovered security groups: [sg-076f0ca74b68addb2 sg-09176f21ae53f5d60] {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:02:12.709Z DEBUG controller.provisioning.cloudprovider Discovered kubernetes version 1.21 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:02:12.770Z DEBUG controller.provisioning.cloudprovider Discovered ami-0413b176c68479e84 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:02:12.931Z DEBUG controller.provisioning.cloudprovider Created launch template, Karpenter-eksworkshop-eksctl-1404659202440619516 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:02:14.748Z INFO controller.provisioning.cloudprovider Launched instance: i-07f4c8c180ece267a, hostname: ip-192-168-25-60.eu-west-1.compute.internal, type: t3a.xlarge, zone: eu-west-1a, capacityType: on-demand {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:02:14.758Z INFO controller.provisioning Created node with 2 pods requesting {"cpu":"2125m","memory":"512M","pods":"5"} from types t3a.xlarge, c6a.xlarge, c5a.xlarge, c6i.xlarge, t3.xlarge and 333 other(s) {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:02:14.758Z INFO controller.provisioning Waiting for unschedulable pods {"commit": "1f7a67b"} ... ``` There are a few things to highlight from the logs above. The first one is in relation with the Provisioner that was used to make the instance selection. The logs point to the `controller.allocation.provisioner/default` making the selection. We will learn in the next sections how to select and use alternative Provisioners. -In the scenario above, the log shows the instance selected was an `on-demand` instance of type **t3a.xlarge** and it was considered from the instance diversified selection: c5a.xlarge c5d.xlarge c3.xlarge c4.xlarge c5ad.xlarge c5.xlarge c6i.xlarge c1.xlarge c5n.xlarge m1.xlarge m3.xlarge t3.xlarge t3a.xlarge m5dn.xlarge m5ad.xlarge m4.xlarge m5zn.xlarge m5n.xlarge m6i.xlarge m5d.xlarge. All the instances in the list are of type `amd64` or x86_64 and all of them are of the right size to bin-pack the deployment of 2 replicas we just did. +In the scenario above, the log shows the instance selected was an `on-demand` instance of type **t3a.xlarge** and it was considered from the instance diversified selection: t3a.xlarge, c6a.xlarge, c5a.xlarge, c6i.xlarge, t3.xlarge and 333 other(s). All the instances in the list are of type `amd64` or x86_64 and all of them are of the right size to at least fit the deployment of 2 replicas . Let's understand first why the instance selected was `on-demand`. As we stated before NodeSelectors are an opt-in mechanism which allow users to specify the nodes on which a pod can be scheduled. Karpenter uses the NodeSelectors defined in the pending pods and provisions capacity accordingly. In this case, the `inflate-amd64` deployment did not state any NodeSelector (like `spot` or `on-demand`) for the `kubernetes.sh/capacity-type`. In these situations Karpenter reverts to the default value for that well-known label. In the case of `kubernetes.sh/capacity-type` Karpenter uses `on-demand` as the default option. @@ -202,21 +206,20 @@ The output should show something similar to the lines below ```bash ... -2022-05-12T04:20:51.896Z INFO controller Batched 2 pod(s) in 1.014499438s {"commit": "00661aa"} -2022-05-12T04:20:52.665Z DEBUG controller Discovered 401 EC2 instance types {"commit": "00661aa"} -2022-05-12T04:20:52.790Z DEBUG controller Discovered EC2 instance types zonal offerings {"commit": "00661aa"} -2022-05-12T04:20:52.960Z DEBUG controller Discovered subnets: [subnet-0204b1b3b885ca98d (eu-west-1a) subnet-037d1d97a6a473fd1 (eu-west-1b) subnet-04c2ca248972479e7 (eu-west-1b) subnet-063d5c7ba912986d5 (eu-west-1a)] {"commit": "00661aa"} -2022-05-12T04:20:53.055Z DEBUG controller Discovered security groups: [sg-03ab1d5d49b00b596 sg-06e7e2ca961ab3bed]{"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:20:53.071Z DEBUG controller Discovered kubernetes version 1.21 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:20:53.140Z DEBUG controller Discovered ami-05dc8c3028bc33fd6 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2-arm64/recommended/image_id" {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:20:53.310Z DEBUG controller Created launch template, Karpenter-eksworkshop-eksctl-13655185630813568172 {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:20:55.201Z INFO controller Launched instance: i-013c73aa33c952ace, hostname: ip-192-168-60-157.eu-west-1.compute.internal, type: c6g.xlarge, zone: eu-west-1b, capacityType: spot {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:20:55.214Z INFO controller Created node with 2 pods requesting {"cpu":"2125m","memory":"512M","pods":"5"} from types c6g.xlarge {"commit": "00661aa", "provisioner": "default"} -2022-05-12T04:20:55.229Z INFO controller Waiting for unschedulable pods {"commit": "00661aa"} +2022-07-01T04:04:37.067Z INFO controller.provisioning Found 2 provisionable pod(s) {"commit": "1f7a67b"} +2022-07-01T04:04:37.067Z INFO controller.provisioning Computed 1 new node(s) will fit 2 pod(s) {"commit": "1f7a67b"} +2022-07-01T04:04:37.164Z DEBUG controller.provisioning.cloudprovider Discovered subnets: [subnet-0e528fbbaf13542c2 (eu-west-1b) subnet-0a9bd9b668d8ae58d (eu-west-1a) subnet-03aec03eee186dc42 (eu-west-1a) subnet-03ff683f2535bcd8d (eu-west-1b)] {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:04:37.209Z DEBUG controller.provisioning.cloudprovider Discovered security groups: [sg-076f0ca74b68addb2 sg-09176f21ae53f5d60] {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:04:37.210Z DEBUG controller.provisioning.cloudprovider Discovered kubernetes version 1.21 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:04:37.249Z DEBUG controller.provisioning.cloudprovider Discovered ami-0efd3ecfd285da630 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2-arm64/recommended/image_id" {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:04:37.405Z DEBUG controller.provisioning.cloudprovider Created launch template, Karpenter-eksworkshop-eksctl-2137869187457706438 {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:04:39.251Z INFO controller.provisioning.cloudprovider Launched instance: i-0adfab70cb7801392, hostname: ip-192-168-81-44.eu-west-1.compute.internal, type: c6g.xlarge, zone: eu-west-1a, capacityType: spot {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:04:39.258Z INFO controller.provisioning Created node with 2 pods requesting {"cpu":"2125m","memory":"512M","pods":"5"} from types c6g.xlarge {"commit": "1f7a67b", "provisioner": "default"} +2022-07-01T04:04:39.258Z INFO controller.provisioning Waiting for unschedulable pods {"commit": "1f7a67b"} ... ``` -As in the previous step with the `inflate-amd64`, the instance selected was `on-demand` but unlike the previous time, the well-known lable selected the instance type to use. +Unlike in the previous step with the `inflate-amd64`, the instance selected was `spot` and the well-known lable selected the instance type to use. {{% notice tip %}} We leave as an optional exersice the following setup. What would happen if you change the deployment and remove the `nodeSelector` section that defines the line `node.kubernetes.io/instance-type: c6g.xlarge`. If you want to try changing the deployment and checking what happens, first scale down the replicas to 0, re-deploy the `inflate-amd64` deployment, and then scale up to 2 pods. What is the difference ? What is the diversified selection now ? @@ -264,6 +267,6 @@ In this section we have learned: * Karpenter uses well-known labels in the NodeSelector pods to Override the type instance selected. In this section we used the NodeSelector `kubernetes.io/arch` to select instances of type `amd64` x86_64 and `arm64`. We also learned that we can select a specific instance type by using the well-known lable `node.kubernetes.io/instance-type` (i.e **c6g.xlarge**). -* When NodeSelectors are not specified, Karpenter will revert to the default configuration setup for that label. In this case, the property for `kubernetes.sh/capacity-type` was not set, meaning that `spot` instances were selected even if the `default` provisioner supports both `spot` and `on-demand`. +* When NodeSelectors are not specified, Karpenter will revert to the default configuration setup for that label. In this case, the property for `kubernetes.sh/capacity-type` was not set in the case of the `arm64` deployment, meaning that `spot` instances were selected even if the `default` provisioner supports both `spot` and `on-demand`. * Karpenter scales `on-demand` instances using a diversified selection as well. Similar to Spot, instances are chosen by the ability of those to bin-pack well the pending pods. Karpenter uses the OnDemand allocation strategy **[lowest-price](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-configuration-strategies.html)** to select which instance to pick from the those with available capacity. diff --git a/content/karpenter/040_karpenter/set_up_the_environment.md b/content/karpenter/040_karpenter/set_up_the_environment.md index 6966d475..9d9ec419 100644 --- a/content/karpenter/040_karpenter/set_up_the_environment.md +++ b/content/karpenter/040_karpenter/set_up_the_environment.md @@ -12,7 +12,7 @@ Before we install Karpenter, there are a few things that we will need to prepare Instances launched by Karpenter must run with an InstanceProfile that grants permissions necessary to run containers and configure networking. Karpenter discovers the InstanceProfile using the name `KarpenterNodeRole-${ClusterName}`. ``` -export KARPENTER_VERSION=v0.10.0 +export KARPENTER_VERSION=v0.13.1 echo "export KARPENTER_VERSION=${KARPENTER_VERSION}" >> ~/.bash_profile TEMPOUT=$(mktemp) curl -fsSL https://karpenter.sh/"${KARPENTER_VERSION}"/getting-started/getting-started-with-eksctl/cloudformation.yaml > $TEMPOUT \ @@ -24,7 +24,7 @@ curl -fsSL https://karpenter.sh/"${KARPENTER_VERSION}"/getting-started/getting-s ``` {{% notice tip %}} -This step may take about 2 minutes. In the meantime, you can [download the file](https://karpenter.sh/v0.10.0/getting-started/getting-started-with-eksctl/cloudformation.yaml) and check the content of the CloudFormation Stack. Check how the stack defines a policy, a role and and Instance profile that will be used to associate to the instances launched. You can also head to the **CloudFormation** console and check which resources does the stack deploy. +This step may take about 2 minutes. In the meantime, you can [download the file](https://karpenter.sh/v0.13.1/getting-started/getting-started-with-eksctl/cloudformation.yaml) and check the content of the CloudFormation Stack. Check how the stack defines a policy, a role and and Instance profile that will be used to associate to the instances launched. You can also head to the **CloudFormation** console and check which resources does the stack deploy. {{% /notice %}} Second, grant access to instances using the profile to connect to the cluster. This command adds the Karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. diff --git a/content/karpenter/040_karpenter/set_up_the_provisioner.md b/content/karpenter/040_karpenter/set_up_the_provisioner.md index f952bc68..e5d20340 100644 --- a/content/karpenter/040_karpenter/set_up_the_provisioner.md +++ b/content/karpenter/040_karpenter/set_up_the_provisioner.md @@ -31,15 +31,31 @@ spec: - key: karpenter.sh/capacity-type operator: In values: ["spot"] + - key: karpenter.k8s.aws/instance-size + operator: NotIn + values: [nano, micro, small, medium, large] limits: resources: cpu: 1000 - provider: - subnetSelector: - alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} - securityGroupSelector: - alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + memory: 1000Gi ttlSecondsAfterEmpty: 30 + ttlSecondsUntilExpired: 2592000 + providerRef: + name: default +--- +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + subnetSelector: + alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + securityGroupSelector: + alpha.eksctl.io/cluster-name: ${CLUSTER_NAME} + tags: + KarpenerProvisionerName: "default" + NodeType: "karpenter-workshop" + IntentLabel: "apps" EOF ``` @@ -47,20 +63,19 @@ EOF We are asking the provisioner to start all new nodes with a label `intent: apps`. This is to differentiate from the `intent: control-apps` we used in the setup of the On-Demand Managed Node group. {{% /notice %}} -The configuration for this provider is quite simple. We will change in the future the provider. For the moment let's focus in a few of the settings used. +The configuration for the provider is split into two parts. The first one defines the provisioner relevant spec. The second part is defined by the provider implementation, in our case `AWSNodeTemplate` and defines the specific configuration that applies to that cloud provider. The Provisioner configuration is quite simple. During the workshop we will change the Provisioner and even use multiple provisioners. For the moment let's focus in a few of the settings used. -* **Requirements Section**: The [Provisioner CRD](https://karpenter.sh/docs/provisioner-crd/) supports defining node properties like instance type and zone. For example, in response to a label of topology.kubernetes.io/zone=us-east-1c, Karpenter will provision nodes in that availability zone. In this example we are setting the `karpenter.sh/capacity-type` to procure EC2 Spot instances. You can learn which other properties are [available here](https://karpenter.sh/v0.6.5/aws/provisioning/). We will work on a few more during the workshop. +* **Requirements Section**: The [Provisioner CRD](https://karpenter.sh/docs/provisioner-crd/) supports defining node properties like instance type and zone. For example, in response to a label of topology.kubernetes.io/zone=us-east-1c, Karpenter will provision nodes in that availability zone. In this example we are setting the `karpenter.sh/capacity-type` to procure EC2 Spot instances, and `karpenter.k8s.aws/instance-size` to avoid smaller instances. You can learn which other properties are [available here](https://karpenter.sh/v0.13.1/tasks/scheduling/#selecting-nodes). We will work on a few more during the workshop. * **Limits section**: Provisioners can define a limit in the number of CPU's and memory allocated to that particular provisioner and part of the cluster. * **Provider section**: This provisioner uses `securityGroupSelector` and `subnetSelector` to discover resources used to launch nodes. It uses the tags that Karpenter attached to the subnets. * **ttlSecondsAfterEmpty**: value configures Karpenter to terminate empty nodes. This behavior can be disabled by leaving the value undefined. In this case we have set it for a quick demonstration to a value of 30 seconds. -* **Tags**: Provisioners can also define a set of tags that the EC2 instances will have upon creation. This helps to enable accounting and governance at the EC2 level. - - +* **ttlSecondsUntilExpired**: optional parameter. When set it defines when a node will be deleted. This is useful to force new nodes with up to date AMI's. In this example we have set the value to 30 days. +* **Tags**: Provisioners can also define a set of tags that the EC2 instances will have upon creation. This helps to enable accounting and governance at the EC2 level. As you can see this is done through as part of the provider section. {{% notice info %}} -Karpenter has been designed to be generic and support other Cloud and Infrastructure providers. At the moment of writing this workshop (**Karpenter 0.10.0**) main implementation and Provisioner available is on AWS. You can read more about the **[configuration available for the AWS Provisioner here](https://karpenter.sh/v0.10.0/aws/)** +Karpenter has been designed to be generic and support other Cloud and Infrastructure providers. At the moment of writing this workshop (**Karpenter 0.13.1**) main implementation and Provisioner available is on AWS. You can read more about the **[configuration available for the AWS Provisioner here](https://karpenter.sh/v0.13.1/aws/)** {{% /notice %}} ## Displaying Karpenter Logs diff --git a/content/karpenter/040_karpenter/using_alternative_provisioners.md b/content/karpenter/040_karpenter/using_alternative_provisioners.md index 8df0b02e..6f2e0bb4 100644 --- a/content/karpenter/040_karpenter/using_alternative_provisioners.md +++ b/content/karpenter/040_karpenter/using_alternative_provisioners.md @@ -132,19 +132,19 @@ The output of Karpenter should look similar to the one below ``` ... -2022-05-12T04:33:28.625Z INFO controller Batched 4 pod(s) in 1.038819076s {"commit": "00661aa"} -2022-05-12T04:33:29.361Z DEBUG controller Discovered 401 EC2 instance types {"commit": "00661aa"} -2022-05-12T04:33:29.482Z DEBUG controller Discovered EC2 instance types zonal offerings {"commit": "00661aa"} -2022-05-12T04:33:29.638Z DEBUG controller Discovered subnets: [subnet-0204b1b3b885ca98d (eu-west-1a) subnet-037d1d97a6a473fd1 (eu-west-1b) subnet-04c2ca248972479e7 (eu-west-1b) subnet-063d5c7ba912986d5 (eu-west-1a)] {"commit": "00661aa"} -2022-05-12T04:33:29.728Z DEBUG controller Discovered security groups: [sg-03ab1d5d49b00b596 sg-06e7e2ca961ab3bed] {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:29.732Z DEBUG controller Discovered kubernetes version 1.21 {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:29.796Z DEBUG controller Discovered ami-0440c10a3f77514d8 for query "/aws/service/eks/optimized-ami/1.21/amazon-linux-2/recommended/image_id" {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:29.971Z DEBUG controller Created launch template, Karpenter-eksworkshop-eksctl-2228580094236845875 {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:31.885Z INFO controller Launched instance: i-02df8ea1e99895e78, hostname: ip-192- 68-14-13.eu-west-1.compute.internal, type: t3a.xlarge, zone: eu-west-1a, capacityType: on-demand {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:31.896Z INFO controller Created node with 2 pods requesting {"cpu":"2125m","memory":"512M","pods":"4"} from types c4.xlarge, c6a.xlarge, c6i.xlarge, c5.xlarge, c5a.xlarge and 263 other(s) {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:32.443Z INFO controller Launched instance: i-0b6984823f26d5b15, hostname: ip-192-168-39-218.eu-west-1.compute.internal, type: t3a.xlarge, zone: eu-west-1b, capacityType: on-demand {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:32.453Z INFO controller Created node with 2 pods requesting {"cpu":"2125m","memory":"512M","pods":"4"} from types c4.xlarge, c6a.xlarge, c6i.xlarge, c5.xlarge, c5a.xlarge and 267 other(s) {"commit": "00661aa", "provisioner": "team1"} -2022-05-12T04:33:32.464Z INFO controller Waiting for unschedulable pods {"commit": "00661aa"} +2022-07-01T04:12:15.781Z INFO controller.provisioning Found 4 provisionable pod(s) {"commit": "1f7a67b"} +2022-07-01T04:12:15.781Z INFO controller.provisioning Computed 2 new node(s) will fit 4 pod(s) {"commit": "1f7a67b"} +2022-07-01T04:12:15.967Z DEBUG controller.provisioning.cloudprovider Discovered subnets: [subnet-0e528fbbaf13542c2 (eu-west-1b) subnet-0a9bd9b668d8ae58d (eu-west-1a) subnet-03aec03eee186dc42 (eu-west-1a) subnet-03ff683f2535bcd8d (eu-west-1b)] {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:16.063Z DEBUG controller.provisioning.cloudprovider Discovered security groups: [sg-076f0ca74b68addb2 sg-09176f21ae53f5d60] {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:16.071Z DEBUG controller.provisioning.cloudprovider Discovered kubernetes version 1.21 {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:16.179Z DEBUG controller.provisioning.cloudprovider Discovered ami-015933fe34749f648 for query "/aws/service/bottlerocket/aws-k8s-1.21/x86_64/latest/image_id" {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:16.456Z DEBUG controller.provisioning.cloudprovider Created launch template, Karpenter-eksworkshop-eksctl-641081096202606695 {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:17.277Z DEBUG controller.node-state Discovered 531 EC2 instance types {"commit": "1f7a67b", "node": "ip-192-168-25-60.eu-west-1.compute.internal"} +2022-07-01T04:12:17.418Z DEBUG controller.node-state Discovered EC2 instance types zonal offerings {"commit": "1f7a67b", "node": "ip-192-168-25-60.eu-west-1.compute.internal"} +2022-07-01T04:12:18.287Z INFO controller.provisioning.cloudprovider Launched instance: i-0e81a84185e589749, hostname: ip-192-168-37-210.eu-west-1.compute.internal, type: t3a.xlarge, zone: eu-west-1b, capacityType: on-demand {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:18.302Z INFO controller.provisioning.cloudprovider Launched instance: i-03c9fc74527b401f4, hostname: ip-192-168-7-134.eu-west-1.compute.internal, type: t3a.xlarge, zone: eu-west-1a, capacityType: on-demand {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:18.306Z INFO controller.provisioning Created node with 2 pods requesting {"cpu":"2125m","memory":"512M","pods":"4"} from types t3a.xlarge, c6a.xlarge, c5a.xlarge, c6i.xlarge, t3.xlarge and 315 other(s) {"commit": "1f7a67b", "provisioner": "team1"} +2022-07-01T04:12:18.306Z DEBUG controller.events Normal {"commit": "1f7a67b", "object": {"kind":"Pod","namespace":"default","name":"inflate-team1-865b77c748-dp9k5","uid":"5b682809-1ae9-4ed2-85c9-451abc11cf75","apiVersion":"v1","resourceVersion":"43463"}, "reason": "NominatePod", "message": "Pod should schedule on ip-192-168-37-210.eu-west-1.compute.internal"} ... ``` @@ -152,6 +152,9 @@ It's interesting to see how the nodes were selected in different availability zo As you are probably guessing by now, this may have to do with the section of the deployment section for `topologySpreadConstraints:` . Latest versions of Karpenter (>0.4.1) support Kubernetes [Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). You can define one or multiple topologySpreadConstraint to instruct the kube-scheduler how to place each incoming Pod in relation to the existing Pods across your cluster. To all effect the bin-packing is still happening is just that the topologySpreadConstraint is applied and forces us to spread the workload across the available `kubernetes.io/zone` + +Check out as well the details in the log. If you recall the Node template in the provider section for the `team1` Provisioner, used Bottlerocket with a custom additional bootstrapping. You can see how the logs showcase Karpenter creating the `Karpenter-eksworkshop-eksctl-641081096202606695` launch template, and adapting it according to the latest version of the AMI and the bootstrapping added to the configuration. This simlifies significantly the life-cycle management and patching of EC2 Instances. + {{% /expand %}} @@ -179,3 +182,5 @@ In this section we have learned: * Pods can select the Provisioner by setting a nodeSelector with the lable `karpenter.sh/provisioner-name` pointing to the right Provisioner. * Karpenter supports **[topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/)**. Topology Spread constraints instruct the kube-scheduler how to place each incoming Pod in relation to the existing Pods across your cluster. In this scenario we discover how to balance pods across Availability Zones. + +* For EKS, AL2, Ubuntu and Bottlerocket AMI's Karpenter does the heavy-lifting managing the underlying Launch Templates keeping AMI's up to dates. Karpenter also allows us to configure extra bootstrappign parameters without us having to manage Launch Templates, this significanlty simplifies the life-cycle management and patching of EC2 Instances while removing the heavy-lifting required to apply bootstrapping additional parameters. From 4d45e3829b8a95d187e125b130886341bdc46bd6 Mon Sep 17 00:00:00 2001 From: Carlos Manzanedo Rueda Date: Fri, 1 Jul 2022 06:25:47 +0100 Subject: [PATCH 2/2] final changes --- content/karpenter/040_karpenter/automatic_node_provisioning.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/content/karpenter/040_karpenter/automatic_node_provisioning.md b/content/karpenter/040_karpenter/automatic_node_provisioning.md index 08ee5262..945b8a13 100644 --- a/content/karpenter/040_karpenter/automatic_node_provisioning.md +++ b/content/karpenter/040_karpenter/automatic_node_provisioning.md @@ -283,3 +283,5 @@ In this section we have learned: * Karpenter can scale-out from zero when applications have available working pods and scale-in to zero when there are no running jobs or pods. +* Provisioners can be setup to define governance and rules that define how nodes will be provisioned within a cluster partition. We can setup requirements such as `karpenter.sh/capacity-type` to allow on-demand and spot instances or use `karpenter.k8s.aws/instance-size` to filter smaller sizes. The full list of supported labels is available **[here](https://karpenter.sh/v0.13.1/tasks/scheduling/#selecting-nodes)** +