From 75a8623e4f2870ab340c119830431624ce8ed012 Mon Sep 17 00:00:00 2001 From: Justin Garrison Date: Mon, 28 Jun 2021 14:51:38 -0700 Subject: [PATCH 1/4] Add getting started based on fargate --- docs/aws/README.md | 98 +++++++++++++++----------- docs/aws/eks-config.yaml | 41 +++++++++++ docs/aws/karpenter.cloudformation.yaml | 54 -------------- 3 files changed, 97 insertions(+), 96 deletions(-) create mode 100644 docs/aws/eks-config.yaml diff --git a/docs/aws/README.md b/docs/aws/README.md index 1ebfb6031c76..3dac21d26f33 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -3,31 +3,35 @@ This guide will provide a complete Karpenter installation for AWS. These steps are opinionated and may need to be adapted for your use case. +>Note: This guide should take less than 1 hour to complete and should cost less than $.25 + ## Environment ```bash -CLOUD_PROVIDER=aws +export CLUSTER_NAME=$USER-karpenter-demo +export AWS_DEFAULT_REGION=us-west-2 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -CLUSTER_NAME=$USER-karpenter-demo -AWS_DEFAULT_REGION=us-west-2 +LATEST_KARPENTER_VERSION=$(curl -fsSL \ + https://api.github.com/repos/awslabs/karpenter/releases/latest \ + | jq -r '.tag_name') ``` ### Create a Cluster -Create an EKS cluster +The example EKS configuration uses [AWS Fargate](https://aws.amazon.com/fargate/) to run the required Karpenter components, but it is not a requirement if you have your own clusters. +All EC2 instances added to this cluster will be controlled by Karpenter autoscaling. + +>Note: It is not recommended to run Karpenter in a cluster also using the [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler). +You _can_ use Karpenter on clusters that use the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) and/or [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler). + ```bash -eksctl create cluster \ ---name ${CLUSTER_NAME} \ ---node-type m5.large \ ---nodes 1 \ ---nodes-min 1 \ ---nodes-max 10 \ ---managed \ ---with-oidc +curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${LATEST_KARPENTER_VERSION}"/docs/aws/eks-config.yaml \ + | envsubst \ + | eksctl create cluster -f - ``` Tag the cluster subnets with the required tags for Karpenter auto discovery. -Note: If you have a cluster with version 1.18 or below you can skip this step. +>Note: If you are using a cluster with version 1.18 or below you can skip this step. More [detailed here](https://github.com/awslabs/karpenter/issues/404#issuecomment-845283904). ```bash @@ -37,7 +41,7 @@ SUBNET_IDS=$(aws cloudformation describe-stacks \ --output text) aws ec2 create-tags \ - --resources $(echo $SUBNET_IDS | tr ',' '\n') \ + --resources $(echo ${SUBNET_IDS//,/ }) \ --tags Key="kubernetes.io/cluster/${CLUSTER_NAME}",Value= ``` @@ -45,46 +49,39 @@ aws ec2 create-tags \ We recommend using [CloudFormation](https://aws.amazon.com/cloudformation/) and [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) (IRSA) to manage these permissions. For production use, please review and restrict these permissions for your use case. -Note: For IRSA to work your [cluster needs an OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) +>Note: For IRSA to work your [cluster needs an OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) ```bash -OIDC_PROVIDER=$(aws eks describe-cluster \ - --name ${CLUSTER_NAME} \ - --query 'cluster.identity.oidc.issuer' \ - --output text) - # Creates IAM resources used by Karpenter -LATEST_KARPENTER_VERSION=$(curl \ - https://api.github.com/repos/awslabs/karpenter/releases/latest | jq -r '.tag_name') TEMPOUT=$(mktemp) curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${LATEST_KARPENTER_VERSION}"/docs/aws/karpenter.cloudformation.yaml > $TEMPOUT \ && aws cloudformation deploy \ --stack-name Karpenter-${CLUSTER_NAME} \ --template-file ${TEMPOUT} \ --capabilities CAPABILITY_NAMED_IAM \ - --parameter-overrides ClusterName=${CLUSTER_NAME} OpenIDConnectIdentityProvider=${OIDC_PROVIDER/https:\/\//} - -# Adds the karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. -kubectl patch configmap aws-auth -n kube-system --patch "$(cat <<-EOM -data: - mapRoles: | - - rolearn: arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} - username: system:node:{{EC2PrivateDNSName}} - groups: - - system:bootstrappers - - system:nodes -$(kubectl get configmap -n kube-system aws-auth -ojsonpath='{.data.mapRoles}' | sed 's/^/ /') -EOM -)" + --parameter-overrides ClusterName=${CLUSTER_NAME} + +# Add the karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. +eksctl create iamidentitymapping \ + --username system:node:{{EC2PrivateDNSName}} \ + --cluster ${CLUSTER_NAME} \ + --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} \ + --group system:bootstrappers \ + --group system:nodes ``` ### Install Karpenter + +Use [`helm`](https://helm.sh/) to deploy Karpenter to the cluster. +For additional values, see [the helm chart values](https://github.com/awslabs/karpenter/blob/main/charts/karpenter/values.yaml) + +>Note: We created a Kubernetes service account with our cluster so we don't need the helm chart to do that. + ```bash helm repo add karpenter https://awslabs.github.io/karpenter/charts helm repo update -# For additional values, see https://github.com/awslabs/karpenter/blob/main/charts/karpenter/values.yaml -helm upgrade --install karpenter karpenter/karpenter --create-namespace --namespace karpenter \ - --set serviceAccount.annotations.'eks\.amazonaws\.com/role-arn'=arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole-${CLUSTER_NAME} +helm upgrade --install karpenter karpenter/karpenter \ + --namespace karpenter --set serviceAccount.create=false ``` ### (Optional) Enable Verbose Logging @@ -103,17 +100,18 @@ kind: Provisioner metadata: name: default spec: + ttlSeconds: 30 cluster: name: ${CLUSTER_NAME} caBundle: $(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.certificateAuthority.data" --output json) endpoint: $(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) EOF -kubectl get provisioner default -oyaml +kubectl get provisioner default -o yaml ``` ### Create some pods Create some dummy pods and observe logs. -> Note: this will cause EC2 Instances to launch, which will be billed to your AWS Account. + ```bash cat <Note: to avoid additional costs make sure you delete all ec2 instances before deleting the other cluster resources. ```bash helm delete karpenter -n karpenter aws cloudformation delete-stack --stack-name Karpenter-${CLUSTER_NAME} aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ - | grep -i karpenter \ + | grep -i Karpenter-${CLUSTER_NAME} \ | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} ``` diff --git a/docs/aws/eks-config.yaml b/docs/aws/eks-config.yaml new file mode 100644 index 000000000000..a92ccc186ad0 --- /dev/null +++ b/docs/aws/eks-config.yaml @@ -0,0 +1,41 @@ +--- +apiVersion: eksctl.io/v1alpha5 +kind: ClusterConfig +metadata: + name: ${CLUSTER_NAME} + region: ${AWS_DEFAULT_REGION} + +iam: + withOIDC: true + serviceAccounts: + - metadata: + name: karpenter + namespace: karpenter + attachPolicy: + Version: "2012-10-17" + Statement: + - Effect: Allow + Resource: "*" + Action: + # Write Operations + - "ec2:CreateLaunchTemplate" + - "ec2:CreateFleet" + - "ec2:RunInstances" + - "ec2:CreateTags" + - "iam:PassRole" + - "ec2:TerminateInstances" + # Read Operations + - "ec2:DescribeLaunchTemplates" + - "ec2:DescribeInstances" + - "ec2:DescribeSecurityGroups" + - "ec2:DescribeSubnets" + - "ec2:DescribeInstanceTypes" + - "ec2:DescribeInstanceTypeOfferings" + - "ec2:DescribeAvailabilityZones" + - "ssm:GetParameter" + +fargateProfiles: + - name: karpenter + selectors: + - namespace: kube-system + - namespace: karpenter diff --git a/docs/aws/karpenter.cloudformation.yaml b/docs/aws/karpenter.cloudformation.yaml index b46710312688..66d681d37678 100644 --- a/docs/aws/karpenter.cloudformation.yaml +++ b/docs/aws/karpenter.cloudformation.yaml @@ -1,64 +1,10 @@ AWSTemplateFormatVersion: "2010-09-09" Description: Resources used by https://github.com/awslabs/karpenter Parameters: - OpenIDConnectIdentityProvider: - Type: String - Description: "Example oidc.eks.us-west-2.amazonaws.com/id/1234567890" ClusterName: Type: String Description: "EKS cluster name" Resources: - KarpenterControllerRole: - Type: "AWS::IAM::Role" - Properties: - RoleName: !Sub "KarpenterControllerRole-${ClusterName}" - Path: / - AssumeRolePolicyDocument: !Sub | - { - "Version": "2012-10-17", - "Statement": [{ - "Effect": "Allow", - "Principal": { - "Federated": "arn:aws:iam::${AWS::AccountId}:oidc-provider/${OpenIDConnectIdentityProvider}" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "${OpenIDConnectIdentityProvider}:aud": "sts.${AWS::URLSuffix}", - "${OpenIDConnectIdentityProvider}:sub": "system:serviceaccount:karpenter:karpenter" - } - } - }] - } - KarpenterControllerPolicy: - Type: "AWS::IAM::Policy" - Properties: - PolicyName: !Sub "KarpenterControllerPolicy-${ClusterName}" - Roles: - - - Ref: "KarpenterControllerRole" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Resource: "*" - Action: - # Write Operations - - "ec2:CreateLaunchTemplate" - - "ec2:CreateFleet" - - "ec2:RunInstances" - - "ec2:CreateTags" - - "iam:PassRole" - - "ec2:TerminateInstances" - # Read Operations - - "ec2:DescribeLaunchTemplates" - - "ec2:DescribeInstances" - - "ec2:DescribeSecurityGroups" - - "ec2:DescribeSubnets" - - "ec2:DescribeInstanceTypes" - - "ec2:DescribeInstanceTypeOfferings" - - "ec2:DescribeAvailabilityZones" - - "ssm:GetParameter" KarpenterNodeInstanceProfile: Type: "AWS::IAM::InstanceProfile" Properties: From 7afb5fdc65373e96efe08edcb52837673cd5c9b6 Mon Sep 17 00:00:00 2001 From: Justin Garrison Date: Mon, 28 Jun 2021 15:53:59 -0700 Subject: [PATCH 2/4] Removed "Notes" and clarified fargate --- docs/aws/README.md | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/aws/README.md b/docs/aws/README.md index 3dac21d26f33..b5fca531a9ed 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -3,7 +3,7 @@ This guide will provide a complete Karpenter installation for AWS. These steps are opinionated and may need to be adapted for your use case. ->Note: This guide should take less than 1 hour to complete and should cost less than $.25 +> This guide should take less than 1 hour to complete and should cost less than $.25 ## Environment ```bash @@ -17,11 +17,8 @@ LATEST_KARPENTER_VERSION=$(curl -fsSL \ ### Create a Cluster -The example EKS configuration uses [AWS Fargate](https://aws.amazon.com/fargate/) to run the required Karpenter components, but it is not a requirement if you have your own clusters. -All EC2 instances added to this cluster will be controlled by Karpenter autoscaling. - ->Note: It is not recommended to run Karpenter in a cluster also using the [Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler). -You _can_ use Karpenter on clusters that use the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) and/or [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler). +Karpenter can run anywhere, including on self-managed node groups, [managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html), or [AWS Fargate](https://aws.amazon.com/fargate/). +This demo will run Karpenter on Fargate, which means all EC2 instances added to this cluster will be controlled by Karpenter autoscaling. ```bash curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${LATEST_KARPENTER_VERSION}"/docs/aws/eks-config.yaml \ @@ -31,7 +28,7 @@ curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${LATEST_KARPENT Tag the cluster subnets with the required tags for Karpenter auto discovery. ->Note: If you are using a cluster with version 1.18 or below you can skip this step. +> If you are using a cluster with version 1.18 or below you can skip this step. More [detailed here](https://github.com/awslabs/karpenter/issues/404#issuecomment-845283904). ```bash @@ -49,7 +46,7 @@ aws ec2 create-tags \ We recommend using [CloudFormation](https://aws.amazon.com/cloudformation/) and [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) (IRSA) to manage these permissions. For production use, please review and restrict these permissions for your use case. ->Note: For IRSA to work your [cluster needs an OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) +> For IRSA to work your [cluster needs an OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) ```bash # Creates IAM resources used by Karpenter @@ -75,7 +72,7 @@ eksctl create iamidentitymapping \ Use [`helm`](https://helm.sh/) to deploy Karpenter to the cluster. For additional values, see [the helm chart values](https://github.com/awslabs/karpenter/blob/main/charts/karpenter/values.yaml) ->Note: We created a Kubernetes service account with our cluster so we don't need the helm chart to do that. +> We created a Kubernetes service account with our cluster so we don't need the helm chart to do that. ```bash helm repo add karpenter https://awslabs.github.io/karpenter/charts @@ -155,7 +152,7 @@ kubectl delete node $NODE_NAME ``` ### Cleanup ->Note: to avoid additional costs make sure you delete all ec2 instances before deleting the other cluster resources. +> to avoid additional costs make sure you delete all ec2 instances before deleting the other cluster resources. ```bash helm delete karpenter -n karpenter aws cloudformation delete-stack --stack-name Karpenter-${CLUSTER_NAME} From 016c26faa3ff47ab98bcc6dd3dbfdd528c3cfcb2 Mon Sep 17 00:00:00 2001 From: Justin Garrison Date: Mon, 28 Jun 2021 17:01:46 -0700 Subject: [PATCH 3/4] Remove LATEST_ from karpenter version --- docs/aws/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/aws/README.md b/docs/aws/README.md index b5fca531a9ed..1a2ea3e6ac56 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -10,7 +10,7 @@ These steps are opinionated and may need to be adapted for your use case. export CLUSTER_NAME=$USER-karpenter-demo export AWS_DEFAULT_REGION=us-west-2 AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) -LATEST_KARPENTER_VERSION=$(curl -fsSL \ +KARPENTER_VERSION=$(curl -fsSL \ https://api.github.com/repos/awslabs/karpenter/releases/latest \ | jq -r '.tag_name') ``` @@ -21,7 +21,7 @@ Karpenter can run anywhere, including on self-managed node groups, [managed node This demo will run Karpenter on Fargate, which means all EC2 instances added to this cluster will be controlled by Karpenter autoscaling. ```bash -curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${LATEST_KARPENTER_VERSION}"/docs/aws/eks-config.yaml \ +curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${KARPENTER_VERSION}"/docs/aws/eks-config.yaml \ | envsubst \ | eksctl create cluster -f - ``` @@ -51,7 +51,7 @@ For production use, please review and restrict these permissions for your use ca ```bash # Creates IAM resources used by Karpenter TEMPOUT=$(mktemp) -curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${LATEST_KARPENTER_VERSION}"/docs/aws/karpenter.cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${KARPENTER_VERSION}"/docs/aws/karpenter.cloudformation.yaml > $TEMPOUT \ && aws cloudformation deploy \ --stack-name Karpenter-${CLUSTER_NAME} \ --template-file ${TEMPOUT} \ From 05a058683d19e4ea34f35f72ad9826b1bc32747c Mon Sep 17 00:00:00 2001 From: Justin Garrison Date: Tue, 29 Jun 2021 09:58:47 -0700 Subject: [PATCH 4/4] Minor edits and fp profile update --- docs/aws/README.md | 10 ++++++---- docs/aws/eks-config.yaml | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/docs/aws/README.md b/docs/aws/README.md index 1a2ea3e6ac56..a9be1e501a33 100644 --- a/docs/aws/README.md +++ b/docs/aws/README.md @@ -3,7 +3,7 @@ This guide will provide a complete Karpenter installation for AWS. These steps are opinionated and may need to be adapted for your use case. -> This guide should take less than 1 hour to complete and should cost less than $.25 +> This guide should take less than 1 hour to complete and cost less than $.25 ## Environment ```bash @@ -18,7 +18,7 @@ KARPENTER_VERSION=$(curl -fsSL \ ### Create a Cluster Karpenter can run anywhere, including on self-managed node groups, [managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html), or [AWS Fargate](https://aws.amazon.com/fargate/). -This demo will run Karpenter on Fargate, which means all EC2 instances added to this cluster will be controlled by Karpenter autoscaling. +This demo will run Karpenter on Fargate, which means all EC2 instances added to this cluster will be controlled by Karpenter. ```bash curl -fsSL https://raw.githubusercontent.com/awslabs/karpenter/"${KARPENTER_VERSION}"/docs/aws/eks-config.yaml \ @@ -46,7 +46,7 @@ aws ec2 create-tags \ We recommend using [CloudFormation](https://aws.amazon.com/cloudformation/) and [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) (IRSA) to manage these permissions. For production use, please review and restrict these permissions for your use case. -> For IRSA to work your [cluster needs an OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) +> For IRSA to work your cluster needs an [OIDC provider](https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html) ```bash # Creates IAM resources used by Karpenter @@ -147,12 +147,14 @@ kubectl scale deployment inflate --replicas 0 ``` Or you can manually delete the node with + +> Karpenter automatically adds a node finalizer to properly cordon and drain nodes before they are terminated. ```bash kubectl delete node $NODE_NAME ``` ### Cleanup -> to avoid additional costs make sure you delete all ec2 instances before deleting the other cluster resources. +> To avoid additional costs make sure you delete all ec2 instances before deleting the other cluster resources. ```bash helm delete karpenter -n karpenter aws cloudformation delete-stack --stack-name Karpenter-${CLUSTER_NAME} diff --git a/docs/aws/eks-config.yaml b/docs/aws/eks-config.yaml index a92ccc186ad0..fba429b22b2a 100644 --- a/docs/aws/eks-config.yaml +++ b/docs/aws/eks-config.yaml @@ -37,5 +37,7 @@ iam: fargateProfiles: - name: karpenter selectors: - - namespace: kube-system - namespace: karpenter + - name: kube-system + selectors: + - namespace: kube-system