diff --git a/Makefile b/Makefile index 10cdb183086b..220b6aeaa9f1 100644 --- a/Makefile +++ b/Makefile @@ -9,9 +9,10 @@ WITH_GOFLAGS = GOFLAGS=$(GOFLAGS) ## Extra helm options CLUSTER_NAME ?= $(shell kubectl config view --minify -o jsonpath='{.clusters[].name}' | rev | cut -d"/" -f1 | rev) CLUSTER_ENDPOINT ?= $(shell kubectl config view --minify -o jsonpath='{.clusters[].cluster.server}') -HELM_OPTS ?= --set controller.clusterName=${CLUSTER_NAME} \ - --set controller.clusterEndpoint=${CLUSTER_ENDPOINT} \ - --set aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} +HELM_OPTS ?= --set serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN} \ + --set clusterName=${CLUSTER_NAME} \ + --set clusterEndpoint=${CLUSTER_ENDPOINT} \ + --set aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} help: ## Display help @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -49,7 +50,7 @@ licenses: ## Verifies dependency licenses and requires GITHUB_TOKEN to be set golicense hack/license-config.hcl karpenter apply: ## Deploy the controller into your ~/.kube/config cluster - helm template --include-crds karpenter charts/karpenter --namespace karpenter \ + helm template --include-crds karpenter charts/karpenter --namespace karpenter \ $(HELM_OPTS) \ --set controller.image=ko://github.com/aws/karpenter/cmd/controller \ --set webhook.image=ko://github.com/aws/karpenter/cmd/webhook \ @@ -58,7 +59,6 @@ apply: ## Deploy the controller into your ~/.kube/config cluster delete: ## Delete the controller from your ~/.kube/config cluster helm template karpenter charts/karpenter --namespace karpenter \ $(HELM_OPTS) \ - --set serviceAccount.create=false \ | kubectl delete -f - codegen: ## Generate code. Must be run if changes are made to ./pkg/apis/... diff --git a/charts/karpenter/.helmignore b/charts/karpenter/.helmignore new file mode 100644 index 000000000000..0e8a0eb36f4c --- /dev/null +++ b/charts/karpenter/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index 55c7af4b7e6b..88b6d5bed6cb 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -1,6 +1,16 @@ apiVersion: v2 -appVersion: "0.6.1" name: karpenter -description: A Helm chart for https://github.com/aws/karpenter/. +description: A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. type: application -version: "0.6.1" +version: 0.6.1 +appVersion: 0.6.1 +keywords: + - cluster + - node + - scheduler + - autoscaling + - lifecycle +home: https://karpenter.sh/ +icon: https://repository-images.githubusercontent.com/278480393/dab059c8-caa1-4b55-aaa7-3d30e47a5616 +sources: + - https://github.com/aws/karpenter/ diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md index 25cef560c632..3a734c437694 100644 --- a/charts/karpenter/README.md +++ b/charts/karpenter/README.md @@ -1,56 +1,77 @@ # karpenter -A Helm chart for https://github.com/aws/karpenter/. +A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. ![Version: 0.6.1](https://img.shields.io/badge/Version-0.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.1](https://img.shields.io/badge/AppVersion-0.6.1-informational?style=flat-square) +## Documentation + +For full Karpenter documentation please checkout [https://karpenter.sh](https://karpenter.sh/v0.6.1/). + ## Installing the Chart -To install the chart with the release name `karpenter`: +Before the chart can be installed the repo needs to be added to Helm, run the following commands to add the repo. -```console -$ helm repo add karpenter https://charts.karpenter.sh -$ helm repo update -$ helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ - --create-namespace --set serviceAccount.create=false --version 0.6.1 \ - --set controller.clusterName=${CLUSTER_NAME} \ - --set controller.clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ - --wait # for the defaulting webhook to install before creating a Provisioner +```bash +helm repo add karpenter https://charts.karpenter.sh/ +helm repo update ``` -You can follow the detailed installation instruction [here](https://karpenter.sh/docs/getting-started/#install). +You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/v0.6.1/getting-started/#install) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command. + +```bash +helm upgrade --install --namespace karpenter --create-namespace \ + karpenter karpenter/karpenter \ + --version 0.6.1 \ + --set serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN} + --set clusterName=${CLUSTER_NAME} \ + --set clusterEndpoint=${CLUSTER_ENDPOINT} \ + --set aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` ## Values | Key | Type | Default | Description | |-----|------|---------|-------------| -| additionalLabels | object | `{}` | Additional labels to add into metadata | +| additionalAnnotations | object | `{}` | Additional annotations to add into metadata. | +| additionalLabels | object | `{}` | Additional labels to add into metadata. | +| affinity | object | `{"nodeAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":{"nodeSelectorTerms":[{"key":"karpenter.sh/provisioner-name","operator":"DoesNotExist"}]}}}` | Affinity rules for scheduling the pod. | | aws.defaultInstanceProfile | string | `""` | The default instance profile to use when launching nodes on AWS | -| controller.affinity | object | `{}` | Affinity rules for scheduling | -| controller.clusterEndpoint | string | `""` | Cluster endpoint | -| controller.clusterName | string | `""` | Cluster name | -| controller.env | list | `[]` | Additional environment variables to run with | -| controller.image | string | `"public.ecr.aws/karpenter/controller:v0.6.1@sha256:5a0bd78e2f7ada324677e2eb82e53b648593e9de1acf0a8fc84138a1a6be753c"` | Image to use for the Karpenter controller | -| controller.nodeSelector | object | `{}` | Node selectors to schedule to nodes with labels. | -| controller.replicas | int | `1` | | -| controller.resources.limits.cpu | int | `1` | | -| controller.resources.limits.memory | string | `"1Gi"` | | -| controller.resources.requests.cpu | int | `1` | | -| controller.resources.requests.memory | string | `"1Gi"` | | -| controller.tolerations | list | `[]` | Tolerations to schedule to nodes with taints. | -| serviceAccount.annotations | object | `{}` | Annotations to add to the service account (like the ARN of the IRSA role) | -| serviceAccount.create | bool | `true` | Create a service account for the application controller | -| serviceAccount.name | string | `"karpenter"` | Service account name | -| webhook.affinity | object | `{}` | Affinity rules for scheduling | -| webhook.env | list | `[]` | List of environment items to add to the webhook | -| webhook.hostNetwork | bool | `false` | Set to true if using custom CNI on EKS | -| webhook.image | string | `"public.ecr.aws/karpenter/webhook:v0.6.1@sha256:7d75747caeb1ca63da1d68925b961c7a61f40faa76aa678320b2d3e090d1713f"` | Image to use for the webhook | -| webhook.nodeSelector | object | `{}` | Node selectors to schedule to nodes with labels. | -| webhook.port | int | `8443` | | -| webhook.replicas | int | `1` | | -| webhook.resources.limits.cpu | string | `"100m"` | | -| webhook.resources.limits.memory | string | `"50Mi"` | | -| webhook.resources.requests.cpu | string | `"100m"` | | -| webhook.resources.requests.memory | string | `"50Mi"` | | -| webhook.tolerations | list | `[]` | Tolerations to schedule to nodes with taints. | +| clusterEndpoint | string | `""` | Cluster endpoint. | +| clusterName | string | `""` | Cluster name. | +| controller.env | list | `[]` | Additional environment variables for the controller pod. | +| controller.image | string | `"public.ecr.aws/karpenter/controller:v0.6.1@sha256:5a0bd78e2f7ada324677e2eb82e53b648593e9de1acf0a8fc84138a1a6be753c"` | Controller image. | +| controller.logLevel | string | `""` | Controller log level, defaults to the global log level | +| controller.resources | object | `{"limits":{"cpu":1,"memory":"1Gi"},"requests":{"cpu":1,"memory":"1Gi"}}` | Resources for the controller pod. | +| controller.securityContext | object | `{}` | SecurityContext for the controller container. | +| fullnameOverride | string | `""` | Overrides the chart's computed fullname. | +| hostNetwork | bool | `false` | Bind the pod to the host network. This is required when using a custom CNI. | +| imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for Docker images. | +| imagePullSecrets | list | `[]` | Image pull secrets for Docker images. | +| logLevel | string | `"info"` | Global log level | +| nameOverride | string | `""` | Overrides the chart's name. | +| nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selectors to schedule the pod to nodes with labels. | +| podAnnotations | object | `{}` | Additional annotations for the pod. | +| podLabels | object | `{}` | Additional labels for the pod. | +| podSecurityContext | object | `{"fsGroup":1000}` | SecurityContext for the pod. | +| priorityClassName | string | `"system-cluster-critical"` | PriorityClass name for the pod. | +| replicas | int | `1` | Number of replicas. | +| serviceAccount.annotations | object | `{}` | Additional annotations for the ServiceAccount. | +| serviceAccount.create | bool | `true` | Specifies if a ServiceAccount should be created. | +| serviceAccount.name | string | `""` | The name of the ServiceAccount to use. If not set and create is true, a name is generated using the fullname template. | +| serviceMonitor.additionalLabels | object | `{}` | Additional labels for the ServiceMonitor. | +| serviceMonitor.enabled | bool | `false` | Specifies whether a ServiceMonitor should be created. | +| serviceMonitor.interval | string | `"1m"` | Scrape interval for the ServiceMonitor. | +| strategy | object | `{"type":"Recreate"}` | Strategy for updating the pod. | +| terminationGracePeriodSeconds | string | `nil` | Override the default termination grace period for the pod. | +| tolerations | list | `[]` | Tolerations to allow the pod to be scheduled to nodes with taints. | +| webhook.env | list | `[]` | Additional environment variables for the webhook pod. | +| webhook.image | string | `"public.ecr.aws/karpenter/webhook:v0.6.1@sha256:7d75747caeb1ca63da1d68925b961c7a61f40faa76aa678320b2d3e090d1713f"` | Webhook image. | +| webhook.logLevel | string | `""` | Webhook log level, defaults to the global log level | +| webhook.port | int | `8443` | The container port to use for the webhook. | +| webhook.resources | object | `{"limits":{"cpu":"100m","memory":"50Mi"},"requests":{"cpu":"100m","memory":"50Mi"}}` | Resources for the webhook pod. | +| webhook.securityContext | object | `{}` | SecurityContext for the webhook container. | +---------------------------------------------- +Autogenerated from chart metadata using [helm-docs v1.7.0](https://github.com/norwoodj/helm-docs/releases/v1.7.0) diff --git a/charts/karpenter/README.md.gotmpl b/charts/karpenter/README.md.gotmpl index 89f8698d7df5..fe4ba48c973c 100644 --- a/charts/karpenter/README.md.gotmpl +++ b/charts/karpenter/README.md.gotmpl @@ -3,24 +3,34 @@ {{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} +## Documentation + +For full Karpenter documentation please checkout [https://karpenter.sh](https://karpenter.sh/v{{ template "chart.version" . }}/). + ## Installing the Chart -To install the chart with the release name `karpenter`: +Before the chart can be installed the repo needs to be added to Helm, run the following commands to add the repo. -```console -$ helm repo add karpenter https://charts.karpenter.sh -$ helm repo update -$ helm upgrade --install karpenter karpenter/{{ template "chart.name" . }} --namespace karpenter \ - --create-namespace --set serviceAccount.create=false --version {{ template "chart.version" . }} \ - --set controller.clusterName=${CLUSTER_NAME} \ - --set controller.clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ - --wait # for the defaulting webhook to install before creating a Provisioner +```bash +helm repo add karpenter https://charts.karpenter.sh/ +helm repo update ``` -You can follow the detailed installation instruction [here](https://karpenter.sh/docs/getting-started/#install). +You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/v{{ template "chart.version" . }}/getting-started/#install) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command. + +```bash +helm upgrade --install --namespace karpenter --create-namespace \ + karpenter karpenter/{{ template "chart.name" . }} \ + --version {{ template "chart.version" . }} \ + --set serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN} + --set clusterName=${CLUSTER_NAME} \ + --set clusterEndpoint=${CLUSTER_ENDPOINT} \ + --set aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ + --wait # for the defaulting webhook to install before creating a Provisioner +``` {{ template "chart.requirementsSection" . }} {{ template "chart.valuesSection" . }} -{{ template "helm-docs.versionFooter" . }} \ No newline at end of file +{{ template "helm-docs.versionFooter" . }} diff --git a/charts/karpenter/templates/_helpers.tpl b/charts/karpenter/templates/_helpers.tpl index 72bff8769c7b..e6305c711dba 100644 --- a/charts/karpenter/templates/_helpers.tpl +++ b/charts/karpenter/templates/_helpers.tpl @@ -2,8 +2,8 @@ Expand the name of the chart. */}} {{- define "karpenter.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} {{/* Create a default fully qualified app name. @@ -11,46 +11,44 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this If release name contains chart name it will be used as a full name. */}} {{- define "karpenter.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} {{/* Create chart name and version as used by the chart label. */}} {{- define "karpenter.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} {{/* -Generate basic labels +Common labels */}} -{{- define "karpenter.labels" }} +{{- define "karpenter.labels" -}} helm.sh/chart: {{ include "karpenter.chart" . }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -app.kubernetes.io/component: karpenter -app.kubernetes.io/part-of: {{ template "karpenter.name" . }} -{{- include "karpenter.selectorLabels" . }} -{{- if .Chart.Version }} -app.kubernetes.io/version: {{ .Chart.Version | quote }} +{{ include "karpenter.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} -{{- if .Values.additionalLabels }} -{{ toYaml .Values.additionalLabels }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.additionalLabels }} +{{ toYaml . }} {{- end }} {{- end }} {{/* Selector labels */}} -{{- define "karpenter.selectorLabels" }} +{{- define "karpenter.selectorLabels" -}} app.kubernetes.io/name: {{ include "karpenter.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end }} @@ -59,9 +57,9 @@ app.kubernetes.io/instance: {{ .Release.Name }} Create the name of the service account to use */}} {{- define "karpenter.serviceAccountName" -}} -{{- if .Values.serviceAccount.enabled -}} - {{ default (include "karpenter.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "karpenter.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml new file mode 100644 index 000000000000..1a364bf0476c --- /dev/null +++ b/charts/karpenter/templates/clusterrole.yaml @@ -0,0 +1,41 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +rules: + - apiGroups: ["karpenter.sh"] + resources: ["provisioners"] + verbs: ["get", "list", "watch"] + - apiGroups: ["karpenter.sh"] + resources: ["provisioners/status"] + verbs: ["create", "delete", "patch", "get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes", "pods"] + verbs: ["get", "list", "watch", "patch", "delete"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["create"] + - apiGroups: [""] + resources: ["pods/binding", "pods/eviction"] + verbs: ["create"] + - apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["list", "watch"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] + verbs: ["get", "watch", "list", "update"] diff --git a/charts/karpenter/templates/clusterrolebinding.yaml b/charts/karpenter/templates/clusterrolebinding.yaml new file mode 100644 index 000000000000..4eb4280b0975 --- /dev/null +++ b/charts/karpenter/templates/clusterrolebinding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "karpenter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/karpenter/templates/100-config-logging.yaml b/charts/karpenter/templates/configmap-logging.yaml similarity index 67% rename from charts/karpenter/templates/100-config-logging.yaml rename to charts/karpenter/templates/configmap-logging.yaml index 8361ff92c75a..7375a0423204 100644 --- a/charts/karpenter/templates/100-config-logging.yaml +++ b/charts/karpenter/templates/configmap-logging.yaml @@ -2,14 +2,17 @@ apiVersion: v1 kind: ConfigMap metadata: name: config-logging - namespace: {{ .Release.Namespace }} labels: - {{- include "karpenter.labels" . | indent 4 }} + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} data: # https://github.com/uber-go/zap/blob/aa3e73ec0896f8b066ddf668597a02f89628ee50/config.go zap-logger-config: | { - "level": "info", + "level": "{{ .Values.logLevel }}", "development": false, "disableStacktrace": true, "disableCaller": true, @@ -31,6 +34,9 @@ data: "timeEncoder": "iso8601" } } - # Log level overrides - # loglevel.controller: info # debug - # loglevel.webhook: info # debug +{{- with .Values.controller.logLevel }} + loglevel.controller: {{ . | quote }} +{{- end }} +{{- with .Values.webhook.logLevel }} + loglevel.webhook: {{ . | quote }} +{{- end }} diff --git a/charts/karpenter/templates/controller/deployment.yaml b/charts/karpenter/templates/controller/deployment.yaml deleted file mode 100644 index 19c0e239706e..000000000000 --- a/charts/karpenter/templates/controller/deployment.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: karpenter-metrics - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -spec: - ports: - - name: metrics - port: 8080 - targetPort: metrics - selector: - karpenter: controller ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: karpenter-controller - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -spec: - replicas: {{ .Values.controller.replicas }} - strategy: - type: Recreate - selector: - matchLabels: - karpenter: controller - template: - metadata: - labels: - karpenter: controller - {{- include "karpenter.labels" . | indent 8 }} - spec: - priorityClassName: system-cluster-critical - serviceAccountName: {{ template "karpenter.serviceAccountName" . }} - containers: - - name: manager - image: {{ .Values.controller.image }} - {{- if .Values.controller.resources }} - resources: {{ toYaml .Values.controller.resources | nindent 12 }} - {{- end }} - ports: - - name: metrics - containerPort: 8080 - - name: health-probe - containerPort: 8081 - livenessProbe: - httpGet: - path: /healthz - port: 8081 - readinessProbe: - httpGet: - path: /readyz - port: 8081 - env: - - name: CLUSTER_NAME - value: {{ .Values.controller.clusterName }} - - name: CLUSTER_ENDPOINT - value: {{ .Values.controller.clusterEndpoint }} - - name: SYSTEM_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - {{- if .Values.aws.defaultInstanceProfile }} - - name: AWS_DEFAULT_INSTANCE_PROFILE - value: {{ .Values.aws.defaultInstanceProfile }} - {{- end }} - {{- with .Values.controller.env }} - {{- toYaml . | nindent 12 }} - {{- end }} - # https://github.com/aws/amazon-eks-pod-identity-webhook/issues/8#issuecomment-636888074 - not needed in k8s versions 1.19+ - securityContext: - fsGroup: 1000 - {{- with .Values.controller.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - affinity: - {{- if .Values.controller.affinity }} - {{- toYaml .Values.controller.affinity | nindent 8 }} - {{- else }} - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - - key: karpenter.sh/provisioner-name - operator: DoesNotExist - {{- end }} - {{- with .Values.controller.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/karpenter/templates/controller/rbac.yaml b/charts/karpenter/templates/controller/rbac.yaml deleted file mode 100644 index 8bf04d27c02d..000000000000 --- a/charts/karpenter/templates/controller/rbac.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: karpenter-controller - labels: - {{- include "karpenter.labels" . | indent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: karpenter-controller -subjects: -- kind: ServiceAccount - name: {{ .Values.serviceAccount.name }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: karpenter-controller - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: karpenter-controller -subjects: -- kind: ServiceAccount - name: {{ template "karpenter.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: karpenter-controller - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -rules: -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -- apiGroups: [""] - resources: ["configmaps/status"] - verbs: ["get", "update", "patch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["create"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create", "get", "patch", "update", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: karpenter-controller - labels: - {{- include "karpenter.labels" . | indent 4 }} -rules: -- apiGroups: ["karpenter.sh"] - resources: ["provisioners"] - verbs: ["get", "list", "watch"] -- apiGroups: ["karpenter.sh"] - resources: ["provisioners/status"] - verbs: ["create", "delete", "patch", "get", "list", "watch"] -- apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: ["storage.k8s.io"] - resources: ["storageclasses"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["nodes", "pods"] - verbs: ["get", "list", "watch", "patch", "delete"] -- apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["create"] -- apiGroups: [""] - resources: ["pods/binding", "pods/eviction"] - verbs: ["create"] -- apiGroups: ["apps"] - resources: ["daemonsets"] - verbs: ["list", "watch"] ---- diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml new file mode 100644 index 000000000000..2f7dde26c803 --- /dev/null +++ b/charts/karpenter/templates/deployment.yaml @@ -0,0 +1,145 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +spec: + replicas: {{ .Values.replicas }} + {{- with .Values.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "karpenter.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "karpenter.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "karpenter.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} + {{- with .Values.terminationGracePeriodSeconds }} + terminationGracePeriodSeconds: {{ . }} + {{- end }} + {{- if .Values.webhook.hostNetwork }} + hostNetwork: true + {{- end }} + containers: + - name: controller + {{- with .Values.controller.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ .Values.controller.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + env: + - name: CLUSTER_NAME + value: {{ .Values.clusterName }} + - name: CLUSTER_ENDPOINT + value: {{ .Values.clusterEndpoint }} + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.aws.defaultInstanceProfile }} + - name: AWS_DEFAULT_INSTANCE_PROFILE + value: {{ .Values.aws.defaultInstanceProfile }} + {{- end }} + {{- with .Values.controller.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: http + containerPort: 8081 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + readinessProbe: + httpGet: + path: /readyz + port: http + {{- with .Values.controller.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + - name: webhook + {{- with .Values.webhook.securityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + image: {{ .Values.webhook.image }} + imagePullPolicy: {{ .Values.imagePullPolicy }} + env: + - name: CLUSTER_NAME + value: {{ .Values.clusterName }} + - name: CLUSTER_ENDPOINT + value: {{ .Values.clusterEndpoint }} + - name: SYSTEM_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.aws.defaultInstanceProfile }} + - name: AWS_DEFAULT_INSTANCE_PROFILE + value: {{ .Values.aws.defaultInstanceProfile }} + {{- end }} + {{- with .Values.webhook.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + args: + - -port={{ .Values.webhook.port }} + ports: + - name: https-webhook + containerPort: {{ .Values.webhook.port }} + protocol: TCP + livenessProbe: + httpGet: + port: https-webhook + scheme: HTTPS + readinessProbe: + httpGet: + port: https-webhook + scheme: HTTPS + {{- with .Values.webhook.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/karpenter/templates/role.yaml b/charts/karpenter/templates/role.yaml new file mode 100644 index 000000000000..1f8e6495e4d4 --- /dev/null +++ b/charts/karpenter/templates/role.yaml @@ -0,0 +1,36 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + - apiGroups: [""] + resources: ["configmaps/status"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["create"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create", "get", "patch", "update", "watch"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["karpenter-webhook-cert"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "create", "update"] diff --git a/charts/karpenter/templates/rolebinding.yaml b/charts/karpenter/templates/rolebinding.yaml new file mode 100644 index 000000000000..595b62793873 --- /dev/null +++ b/charts/karpenter/templates/rolebinding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "karpenter.fullname" . }} +subjects: + - kind: ServiceAccount + name: {{ template "karpenter.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} diff --git a/charts/karpenter/templates/secret-webhook-cert.yaml b/charts/karpenter/templates/secret-webhook-cert.yaml new file mode 100644 index 000000000000..05a4c73bce84 --- /dev/null +++ b/charts/karpenter/templates/secret-webhook-cert.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "karpenter.fullname" . }}-webhook--cert + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +data: {} # Injected by karpenter-webhook diff --git a/charts/karpenter/templates/service.yaml b/charts/karpenter/templates/service.yaml new file mode 100644 index 000000000000..173d08aa20ce --- /dev/null +++ b/charts/karpenter/templates/service.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +spec: + type: ClusterIP + ports: + - name: http-metrics + port: 8080 + targetPort: http-metrics + protocol: TCP + - name: https-webhook + port: 443 + targetPort: https-webhook + protocol: TCP + selector: + {{- include "karpenter.selectorLabels" . | nindent 4 }} diff --git a/charts/karpenter/templates/serviceaccount.yaml b/charts/karpenter/templates/serviceaccount.yaml index aa2a1e001511..60176492dfec 100644 --- a/charts/karpenter/templates/serviceaccount.yaml +++ b/charts/karpenter/templates/serviceaccount.yaml @@ -1,13 +1,17 @@ -{{- if .Values.serviceAccount.create }} +{{- if .Values.serviceAccount.create -}} apiVersion: v1 kind: ServiceAccount metadata: name: {{ include "karpenter.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} labels: - {{- include "karpenter.labels" . | indent 4 }} - {{- with .Values.serviceAccount.annotations }} + {{- include "karpenter.labels" . | nindent 4 }} + {{- if or .Values.additionalAnnotations .Values.serviceAccount.annotations }} annotations: + {{- with .Values.additionalAnnotations }} + {{- toYaml . | nindent 4 }} + {{- }} + {{- with .Values.serviceAccount.annotations }} {{- toYaml . | nindent 4 }} {{- end }} -{{- end }} + {{- end }} +{{- end -}} diff --git a/charts/karpenter/templates/servicemonitor.yaml b/charts/karpenter/templates/servicemonitor.yaml new file mode 100644 index 000000000000..3edd2e0a1152 --- /dev/null +++ b/charts/karpenter/templates/servicemonitor.yaml @@ -0,0 +1,28 @@ +{{- if.Values.serviceMonitor.enabled -}} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "karpenter.fullname" . }} + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.serviceMonitor.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +spec: + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- include "karpenter.selectorLabels" . | nindent 6 }} + endpoints: + - port: http-metrics + path: /metrics + {{- with .Values.serviceMonitor.interval }} + interval: {{ . }} + {{- end }} +{{- end -}} diff --git a/charts/karpenter/templates/webhook/deployment.yaml b/charts/karpenter/templates/webhook/deployment.yaml deleted file mode 100644 index 9f86f66f5da9..000000000000 --- a/charts/karpenter/templates/webhook/deployment.yaml +++ /dev/null @@ -1,109 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: karpenter-webhook - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -spec: - ports: - - port: 443 - targetPort: webhook - selector: - karpenter: webhook ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: karpenter-webhook - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -spec: - replicas: {{ .Values.webhook.replicas }} - strategy: - type: Recreate - selector: - matchLabels: - karpenter: webhook - template: - metadata: - labels: - karpenter: webhook - {{- include "karpenter.labels" . | indent 8 }} - spec: - priorityClassName: system-cluster-critical - serviceAccountName: {{ template "karpenter.serviceAccountName" . }} - containers: - - name: webhook - image: {{ .Values.webhook.image }} - args: - - -port={{ .Values.webhook.port }} - {{- if .Values.webhook.resources }} - resources: {{ toYaml .Values.webhook.resources | nindent 12 }} - {{- end }} - ports: - - name: webhook - containerPort: {{ .Values.webhook.port }} - livenessProbe: - httpGet: - scheme: HTTPS - port: {{ .Values.webhook.port }} - readinessProbe: - httpGet: - scheme: HTTPS - port: {{ .Values.webhook.port }} - env: - - name: CLUSTER_NAME - value: {{ .Values.controller.clusterName }} - - name: CLUSTER_ENDPOINT - value: {{ .Values.controller.clusterEndpoint }} - - name: SYSTEM_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - {{- if .Values.aws.defaultInstanceProfile }} - - name: AWS_DEFAULT_INSTANCE_PROFILE - value: {{ .Values.aws.defaultInstanceProfile }} - {{- end }} - {{- with .Values.webhook.env }} - {{- toYaml . | nindent 12 }} - {{- end }} - # https://github.com/aws/amazon-eks-pod-identity-webhook/issues/8#issuecomment-636888074 - not needed in k8s versions 1.19+ - securityContext: - fsGroup: 1000 - {{- with .Values.webhook.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - affinity: - {{- if .Values.webhook.affinity }} - {{- toYaml .Values.webhook.affinity | nindent 8 }} - {{- else }} - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux - - key: karpenter.sh/provisioner-name - operator: DoesNotExist - {{- end }} - {{- if .Values.webhook.hostNetwork }} - hostNetwork: true - {{- end }} - {{- with .Values.webhook.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: karpenter-webhook-cert - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -data: {} # Injected by karpenter-webhook diff --git a/charts/karpenter/templates/webhook/rbac.yaml b/charts/karpenter/templates/webhook/rbac.yaml deleted file mode 100644 index 20305267b4b2..000000000000 --- a/charts/karpenter/templates/webhook/rbac.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: karpenter-webhook - labels: - {{- include "karpenter.labels" . | indent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: karpenter-webhook -subjects: -- kind: ServiceAccount - name: {{ template "karpenter.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: karpenter-webhook - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: karpenter-webhook -subjects: -- kind: ServiceAccount - name: {{ template "karpenter.serviceAccountName" . }} - namespace: {{ .Release.Namespace }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: karpenter-webhook - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | indent 4 }} -rules: -- apiGroups: [""] - resources: ["configmaps", "namespaces"] - verbs: ["get", "list", "watch"] -- apiGroups: [""] - resources: ["secrets"] - resourceNames: ["karpenter-webhook-cert"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list", "watch"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["get", "watch", "create", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: karpenter-webhook - labels: - {{- include "karpenter.labels" . | indent 4 }} -rules: -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] - verbs: ["get", "watch", "list", "update"] ---- diff --git a/charts/karpenter/templates/webhook/webhooks.yaml b/charts/karpenter/templates/webhook/webhooks.yaml deleted file mode 100644 index dd740c59a0dc..000000000000 --- a/charts/karpenter/templates/webhook/webhooks.yaml +++ /dev/null @@ -1,73 +0,0 @@ -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: defaulting.webhook.provisioners.karpenter.sh - labels: - {{- include "karpenter.labels" . | indent 4 }} -webhooks: -- admissionReviewVersions: ["v1"] - clientConfig: - service: - name: karpenter-webhook - namespace: '{{ .Release.Namespace }}' - failurePolicy: Fail - sideEffects: None - name: defaulting.webhook.provisioners.karpenter.sh - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - resources: - - provisioners - - provisioners/status - operations: - - CREATE - - UPDATE ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.provisioners.karpenter.sh - labels: - {{- include "karpenter.labels" . | indent 4 }} -webhooks: -- admissionReviewVersions: ["v1"] - clientConfig: - service: - name: karpenter-webhook - namespace: '{{ .Release.Namespace }}' - failurePolicy: Fail - sideEffects: None - name: validation.webhook.provisioners.karpenter.sh - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1alpha5 - resources: - - provisioners - - provisioners/status - operations: - - CREATE - - UPDATE - - DELETE ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.config.karpenter.sh - labels: - {{- include "karpenter.labels" . | indent 4 }} -webhooks: -- admissionReviewVersions: ["v1"] - clientConfig: - service: - name: karpenter-webhook - namespace: '{{ .Release.Namespace }}' - failurePolicy: Fail - sideEffects: None - name: validation.webhook.config.karpenter.sh - objectSelector: - matchLabels: - app.kubernetes.io/part-of: karpenter diff --git a/charts/karpenter/templates/webhooks.yaml b/charts/karpenter/templates/webhooks.yaml new file mode 100644 index 000000000000..d48733e21f45 --- /dev/null +++ b/charts/karpenter/templates/webhooks.yaml @@ -0,0 +1,77 @@ +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: defaulting.webhook.provisioners.karpenter.sh + labels: + {{- include "karpenter.labels" . | nindent 4 }} + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- }} +webhooks: + - name: defaulting.webhook.provisioners.karpenter.sh + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + failurePolicy: Fail + sideEffects: None + rules: + - apiGroups: + - karpenter.sh + apiVersions: + - v1alpha5 + resources: + - provisioners + - provisioners/status + operations: + - CREATE + - UPDATE +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.webhook.provisioners.karpenter.sh + labels: + {{- include "karpenter.labels" . | nindent 4 }} +webhooks: + - name: validation.webhook.provisioners.karpenter.sh + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + failurePolicy: Fail + sideEffects: None + rules: + - apiGroups: + - karpenter.sh + apiVersions: + - v1alpha5 + resources: + - provisioners + - provisioners/status + operations: + - CREATE + - UPDATE + - DELETE +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validation.webhook.config.karpenter.sh + labels: + {{- include "karpenter.labels" . | nindent 4 }} +webhooks: + - name: validation.webhook.config.karpenter.sh + admissionReviewVersions: ["v1"] + clientConfig: + service: + name: {{ include "karpenter.fullname" . }} + namespace: {{ .Release.Namespace }} + failurePolicy: Fail + sideEffects: None + objectSelector: + matchLabels: + app.kubernetes.io/part-of: {{ template "karpenter.name" . }} diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 06c1b1119a63..c086f8005bf8 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -1,30 +1,95 @@ -# -- Additional labels to add into metadata +# -- Overrides the chart's name. +nameOverride: "" + +# -- Overrides the chart's computed fullname. +fullnameOverride: "" + +# -- Additional labels to add into metadata. additionalLabels: {} # app: karpenter + +# -- Additional annotations to add into metadata. +additionalAnnotations: {} + +# -- Image pull policy for Docker images. +imagePullPolicy: IfNotPresent + +# -- Image pull secrets for Docker images. +imagePullSecrets: [] + serviceAccount: - # -- Create a service account for the application controller + # -- Specifies if a ServiceAccount should be created. create: true - # -- Service account name - name: karpenter - # -- Annotations to add to the service account (like the ARN of the IRSA role) + # -- The name of the ServiceAccount to use. + # If not set and create is true, a name is generated using the fullname template. + name: "" + # -- Additional annotations for the ServiceAccount. annotations: {} + +serviceMonitor: + # -- Specifies whether a ServiceMonitor should be created. + enabled: false + # -- Additional labels for the ServiceMonitor. + additionalLabels: {} + # myLabel: myValue + # -- Scrape interval for the ServiceMonitor. + interval: 1m + +# -- Number of replicas. +replicas: 1 + +# -- Strategy for updating the pod. +strategy: + type: Recreate + +# -- Additional labels for the pod. +podLabels: {} + +# -- Additional annotations for the pod. +podAnnotations: {} + +# -- SecurityContext for the pod. +podSecurityContext: + fsGroup: 1000 + +# -- PriorityClass name for the pod. +priorityClassName: system-cluster-critical + +# -- Override the default termination grace period for the pod. +terminationGracePeriodSeconds: + +# -- Bind the pod to the host network. +# This is required when using a custom CNI. +hostNetwork: false + +# -- Node selectors to schedule the pod to nodes with labels. +nodeSelector: + kubernetes.io/os: linux + +# -- Affinity rules for scheduling the pod. +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - key: karpenter.sh/provisioner-name + operator: DoesNotExist + +# -- Tolerations to allow the pod to be scheduled to nodes with taints. +tolerations: [] + controller: - # -- Additional environment variables to run with - ## - name: AWS_REGION - ## - value: eu-west-1 - env: [] - # -- Node selectors to schedule to nodes with labels. - nodeSelector: {} - # -- Tolerations to schedule to nodes with taints. - tolerations: [] - # -- Affinity rules for scheduling - affinity: {} - # -- Image to use for the Karpenter controller + # -- Controller image. image: "public.ecr.aws/karpenter/controller:v0.6.1@sha256:5a0bd78e2f7ada324677e2eb82e53b648593e9de1acf0a8fc84138a1a6be753c" - # -- Cluster name - clusterName: "" - # -- Cluster endpoint - clusterEndpoint: "" + + # -- SecurityContext for the controller container. + securityContext: {} + + # -- Additional environment variables for the controller pod. + env: [] + # - name: AWS_REGION + # - value: eu-west-1 + + # -- Resources for the controller pod. resources: requests: cpu: 1 @@ -32,29 +97,46 @@ controller: limits: cpu: 1 memory: 1Gi - replicas: 1 + + # -- Controller log level, defaults to the global log level + logLevel: "" + webhook: - # -- List of environment items to add to the webhook - env: [] - # -- Node selectors to schedule to nodes with labels. - nodeSelector: {} - # -- Tolerations to schedule to nodes with taints. - tolerations: [] - # -- Affinity rules for scheduling - affinity: {} - # -- Image to use for the webhook + # -- Webhook image. image: "public.ecr.aws/karpenter/webhook:v0.6.1@sha256:7d75747caeb1ca63da1d68925b961c7a61f40faa76aa678320b2d3e090d1713f" - # -- Set to true if using custom CNI on EKS - hostNetwork: false + + # -- SecurityContext for the webhook container. + securityContext: {} + + # -- The container port to use for the webhook. port: 8443 + + # -- Additional environment variables for the webhook pod. + env: [] + # - name: AWS_REGION + # - value: eu-west-1 + + # -- Resources for the webhook pod. resources: - limits: + requests: cpu: 100m memory: 50Mi - requests: + limits: cpu: 100m memory: 50Mi - replicas: 1 + + # -- Webhook log level, defaults to the global log level + logLevel: "" + +# -- Global log level +logLevel: info + +# -- Cluster name. +clusterName: "" + +# -- Cluster endpoint. +clusterEndpoint: "" + aws: # -- The default instance profile to use when launching nodes on AWS defaultInstanceProfile: "" diff --git a/website/content/en/preview/development-guide.md b/website/content/en/preview/development-guide.md index 2935246847a0..53d236aca4ff 100644 --- a/website/content/en/preview/development-guide.md +++ b/website/content/en/preview/development-guide.md @@ -21,12 +21,13 @@ The following tools are required for contributing to the Karpenter project. Based on how you are running your Kubernetes cluster, follow the [Environment specific setup](#environment-specific-setup) to configure your environment before you continue. Once you have your environment set up, to install Karpenter in the Kubernetes cluster specified in your `~/.kube/config` run the following commands. -``` -CLOUD_PROVIDER= make apply # Install Karpenter +```bash +CLOUD_PROVIDER="" make apply # Install Karpenter make delete # Uninstall Karpenter ``` ### Developer Loop + * Make sure dependencies are installed * Run `make codegen` to make sure yaml manifests are generated * Run `make toolchain` to install cli tools for building and testing the project @@ -34,69 +35,79 @@ make delete # Uninstall Karpenter * Make sure you have valid credentials to your development repository. * `$KO_DOCKER_REPO` must point to your development repository * Your cluster must have permissions to read from the repository -* It's also a good idea to persist `$CLOUD_PROVIDER` in your environment variables to simplify the `make apply` command. +* It's also a good idea to persist `${CLOUD_PROVIDER}` in your environment variables to simplify the `make apply` command. ### Build and Deploy + *Note: these commands do not rely on each other and may be executed independently* -```sh + +```bash make apply # quickly deploy changes to your cluster make dev # run codegen, lint, and tests ``` ### Testing -```sh + +```bash make test # E2e correctness tests make battletest # More rigorous tests run in CI environment ``` ### Verbose Logging -```sh + +```bash kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' ``` ### Debugging Metrics + OSX: -```sh + +```bash open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 ``` Linux: -```sh + +```bash gio open http://localhost:8080/metrics && kubectl port-forward service/karpenter-metrics -n karpenter 8080 ``` ### Tailing Logs + While you can tail Karpenter's logs with kubectl, there's a number of tools out there that enhance the experience. We recommend [Stern](https://pkg.go.dev/github.com/planetscale/stern#section-readme): -```sh -stern -l karpenter=controller -n karpenter +```bash +stern -n karpenter -l app.kubernetes.io/name=karpenter ``` ## Environment specific setup ### AWS + Set the CLOUD_PROVIDER environment variable to build cloud provider specific packages of Karpenter. ```sh -export CLOUD_PROVIDER=aws +export CLOUD_PROVIDER="aws" ``` For local development on Karpenter you will need a Docker repo which can manage your images for Karpenter components. You can use the following command to provision an ECR repository. -```sh + +```bash aws ecr create-repository \ --repository-name karpenter/controller \ --image-scanning-configuration scanOnPush=true \ - --region ${AWS_DEFAULT_REGION} + --region "${AWS_DEFAULT_REGION}" aws ecr create-repository \ --repository-name karpenter/webhook \ --image-scanning-configuration scanOnPush=true \ - --region ${AWS_DEFAULT_REGION} + --region "${AWS_DEFAULT_REGION}" ``` Once you have your ECR repository provisioned, configure your Docker daemon to authenticate with your newly created repository. -```sh +```bash export KO_DOCKER_REPO="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/karpenter" -aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin $KO_DOCKER_REPO +aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${KO_DOCKER_REPO}" ``` diff --git a/website/content/en/preview/getting-started-with-kops/_index.md b/website/content/en/preview/getting-started-with-kops/_index.md index d786e5194765..9eb5b403d801 100644 --- a/website/content/en/preview/getting-started-with-kops/_index.md +++ b/website/content/en/preview/getting-started-with-kops/_index.md @@ -110,7 +110,7 @@ spec: cpu: 1 EOF kubectl scale deployment inflate --replicas 5 -kubectl logs -f -n kube-system -l karpenter=controller +kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller ``` ### Automatic Node Termination @@ -120,7 +120,7 @@ Karpenter should terminate the now empty nodes. ```bash kubectl delete deployment inflate -kubectl logs -f -n karpenter -l karpenter=controller +kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller ``` ### Manual Node Termination @@ -158,4 +158,4 @@ To avoid additional charges, remove the demo infrastructure from your AWS accoun ```bash kops delete cluster $CLUSTER_NAME --yes -``` \ No newline at end of file +``` diff --git a/website/content/en/preview/getting-started-with-terraform/_index.md b/website/content/en/preview/getting-started-with-terraform/_index.md index c6cc6883e5ee..294064706ce5 100644 --- a/website/content/en/preview/getting-started-with-terraform/_index.md +++ b/website/content/en/preview/getting-started-with-terraform/_index.md @@ -36,17 +36,14 @@ Install these tools before proceeding: Login to the AWS CLI with a user that has sufficient privileges to create a cluster. - - - ### Setting up Variables After setting up the tools, set the following environment variables to store commonly used values. ```bash -export CLUSTER_NAME=$USER-karpenter-demo -export AWS_DEFAULT_REGION=us-east-1 +export CLUSTER_NAME="${USER}-karpenter-demo" +export AWS_DEFAULT_REGION="us-east-1" ``` The first thing we need to do is create our `main.tf` file and place the @@ -60,7 +57,6 @@ variable "cluster_name" { } ``` - ### Create a Cluster We're going to use two different Terraform modules to create our cluster - one @@ -118,7 +114,7 @@ EKS cluster. This may take some time. ```bash terraform init -terraform apply -var cluster_name=$CLUSTER_NAME +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` There's a good chance it will fail when trying to configure the aws-auth @@ -128,9 +124,9 @@ configure both your local CLI and Terraform to use the file. Then try the apply again. ```bash -export KUBECONFIG=${PWD}/kubeconfig_${CLUSTER_NAME} -export KUBE_CONFIG_PATH=$KUBECONFIG -terraform apply -var cluster_name=$CLUSTER_NAME +export KUBECONFIG="${PWD}/kubeconfig_${CLUSTER_NAME}" +export KUBE_CONFIG_PATH="${KUBECONFIG}" +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` Everything should apply successfully now! @@ -138,6 +134,7 @@ Everything should apply successfully now! ### Create the EC2 Spot Service Linked Role This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). + ```bash aws iam create-service-linked-role --aws-service-name spot.amazonaws.com # If the role has already been successfully created, you will see: @@ -172,7 +169,7 @@ resource "aws_iam_instance_profile" "karpenter" { Go ahead and apply the changes. ```bash -terraform apply -var cluster_name=$CLUSTER_NAME +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` Now, Karpenter can use this instance profile to launch new EC2 instances and @@ -234,7 +231,7 @@ Then, apply the changes. ```bash terraform init -terraform apply -var cluster_name=$CLUSTER_NAME +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` ### Install Karpenter Helm Chart @@ -260,12 +257,12 @@ resource "helm_release" "karpenter" { } set { - name = "controller.clusterName" + name = "clusterName" value = var.cluster_name } set { - name = "controller.clusterEndpoint" + name = "clusterEndpoint" value = module.eks.cluster_endpoint } @@ -280,14 +277,13 @@ Now, deploy Karpenter by applying the new Terraform config. ```bash terraform init -terraform apply -var cluster_name=$CLUSTER_NAME +terraform apply -var "cluster_name=${CLUSTER_NAME}" ``` ### Enable Debug Logging (optional) -```sh -kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' -``` + +The global log level can be modified with the `logLevel` chart value (e.g. `--set logLevel=debug`) or the individual components can have their log level set with `controller.logLevel` or `webhook.logLevel` chart values. ### Provisioner @@ -365,7 +361,7 @@ spec: cpu: 1 EOF kubectl scale deployment inflate --replicas 5 -kubectl logs -f -n karpenter $(kubectl get pods -n karpenter -l karpenter=controller -o name) +kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller ``` ### Automatic Node Termination @@ -375,7 +371,7 @@ Karpenter should terminate the now empty nodes. ```bash kubectl delete deployment inflate -kubectl logs -f -n karpenter $(kubectl get pods -n karpenter -l karpenter=controller -o name) +kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller ``` ### Manual Node Termination @@ -387,7 +383,7 @@ drained and the instance is terminated. Keep in mind, this only works for nodes provisioned by Karpenter. ```bash -kubectl delete node $NODE_NAME +kubectl delete node "${NODE_NAME}" ``` ## Cleanup @@ -402,9 +398,9 @@ created LaunchTemplates. kubectl delete deployment inflate kubectl delete node -l karpenter.sh/provisioner-name=default helm uninstall karpenter --namespace karpenter -terraform destroy -var cluster_name=$CLUSTER_NAME +terraform destroy -var "cluster_name=${CLUSTER_NAME}" aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ - | grep -i Karpenter-${CLUSTER_NAME} \ + | grep -i "Karpenter-${CLUSTER_NAME}" \ | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} ``` diff --git a/website/content/en/preview/getting-started/_index.md b/website/content/en/preview/getting-started/_index.md index 83be09c42608..bfb54c82723f 100644 --- a/website/content/en/preview/getting-started/_index.md +++ b/website/content/en/preview/getting-started/_index.md @@ -18,11 +18,12 @@ Follow the clean-up instructions to reduce any charges. ## Install -Karpenter is installed in clusters with a helm chart. +Karpenter is installed in clusters with a Helm chart. -Karpenter additionally requires IAM Roles for Service Accounts (IRSA). IRSA -permits Karpenter (within the cluster) to make privileged requests to AWS (as -the cloud provider). +Karpenter requires cloud provider permissions to provision nodes, for AWS IAM +Roles for Service Accounts (IRSA) should be used. IRSA permits Karpenter +(within the cluster) to make privileged requests to AWS (as the cloud provider) +via a ServiceAccount. ### Required Utilities @@ -42,9 +43,9 @@ After setting up the tools, set the following environment variables to store commonly used values. ```bash -export CLUSTER_NAME=$USER-karpenter-demo -export AWS_DEFAULT_REGION=us-west-2 -AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +export CLUSTER_NAME="${USER}-karpenter-demo" +export AWS_DEFAULT_REGION="us-west-2" +AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" ``` ### Create a Cluster @@ -73,6 +74,8 @@ iam: withOIDC: true EOF eksctl create cluster -f cluster.yaml + +export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output text) ``` This guide uses a managed node group to host Karpenter. @@ -92,10 +95,10 @@ TEMPOUT=$(mktemp) curl -fsSL https://karpenter.sh{{< relref "." >}}cloudformation.yaml > $TEMPOUT \ && aws cloudformation deploy \ - --stack-name Karpenter-${CLUSTER_NAME} \ - --template-file ${TEMPOUT} \ + --stack-name "Karpenter-${CLUSTER_NAME}" \ + --template-file "${TEMPOUT}" \ --capabilities CAPABILITY_NAMED_IAM \ - --parameter-overrides ClusterName=${CLUSTER_NAME} + --parameter-overrides "ClusterName=${CLUSTER_NAME}" ``` Second, grant access to instances using the profile to connect to the cluster. This command adds the Karpenter node role to your aws-auth configmap, allowing nodes with this role to connect to the cluster. @@ -103,8 +106,8 @@ Second, grant access to instances using the profile to connect to the cluster. T ```bash eksctl create iamidentitymapping \ --username system:node:{{EC2PrivateDNSName}} \ - --cluster ${CLUSTER_NAME} \ - --arn arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME} \ + --cluster "${CLUSTER_NAME}" \ + --arn "arn:aws:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}" \ --group system:bootstrappers \ --group system:nodes ``` @@ -115,16 +118,21 @@ Now, Karpenter can launch new EC2 instances and those instances can connect to y Karpenter requires permissions like launching instances. This will create an AWS IAM Role, Kubernetes service account, and associate them using [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html). -``` +```bash eksctl create iamserviceaccount \ - --cluster $CLUSTER_NAME --name karpenter --namespace karpenter \ - --attach-policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/KarpenterControllerPolicy-$CLUSTER_NAME \ + --cluster "${CLUSTER_NAME}" --name karpenter --namespace karpenter \ + --role-name "${CLUSTER_NAME}-karpenter" \ + --attach-policy-arn "arn:aws:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME}" \ + --role-only \ --approve + +export KARPENTER_IAM_ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" ``` ### Create the EC2 Spot Service Linked Role This step is only necessary if this is the first time you're using EC2 Spot in this account. More details are available [here](https://docs.aws.amazon.com/batch/latest/userguide/spot_fleet_IAM_role.html). + ```bash aws iam create-service-linked-role --aws-service-name spot.amazonaws.com # If the role has already been successfully created, you will see: @@ -133,26 +141,31 @@ aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ### Install Karpenter Helm Chart -Use helm to deploy Karpenter to the cluster. +Use Helm to deploy Karpenter to the cluster. -We created a Kubernetes service account when we created the cluster using -eksctl. Thus, we don't need the helm chart to do that. +Before the chart can be installed the repo needs to be added to Helm, run the following commands to add the repo. ```bash -helm repo add karpenter https://charts.karpenter.sh +helm repo add karpenter https://charts.karpenter.sh/ helm repo update -helm upgrade --install karpenter karpenter/karpenter --namespace karpenter \ - --create-namespace --set serviceAccount.create=false --version {{< param "latest_release_version" >}} \ - --set controller.clusterName=${CLUSTER_NAME} \ - --set controller.clusterEndpoint=$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output json) \ +``` + +Install the chart passing in the cluster details and the Karpenter role ARN. + +```bash +helm upgrade --install --namespace karpenter --create-namespace \ + karpenter karpenter/karpenter \ + --version {{< param "latest_release_version" >}} \ + --set serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN} + --set clusterName=${CLUSTER_NAME} \ + --set clusterEndpoint=${CLUSTER_ENDPOINT} \ --set aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ --wait # for the defaulting webhook to install before creating a Provisioner ``` ### Enable Debug Logging (optional) -```sh -kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' -``` + +The global log level can be modified with the `logLevel` chart value (e.g. `--set logLevel=debug`) or the individual components can have their log level set with `controller.logLevel` or `webhook.logLevel` chart values. ### Create Grafana dashboards (optional) @@ -162,7 +175,7 @@ The Karpenter repo contains multiple [importable dashboards](https://github.com/ The following commands will deploy a Prometheus and Grafana stack that is suitable for this guide but does not include persistent storage or other configurations that would be necessary for monitoring a production deployment of Karpenter. -```sh +```bash helm repo add grafana-charts https://grafana.github.io/helm-charts helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update @@ -178,13 +191,13 @@ helm install --namespace monitoring grafana grafana-charts/grafana --values graf The Grafana instance may be accessed using port forwarding. -```sh +```bash kubectl port-forward --namespace monitoring svc/grafana 3000:80 ``` The new stack has only one user, `admin`, and the password is stored in a secret. The following command will retrieve the password. -```sh +```bash kubectl get secret --namespace monitoring grafana -o jsonpath="{.data.admin-password}" | base64 --decode ``` @@ -265,7 +278,7 @@ spec: cpu: 1 EOF kubectl scale deployment inflate --replicas 5 -kubectl logs -f -n karpenter $(kubectl get pods -n karpenter -l karpenter=controller -o name) +kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller ``` ### Automatic Node Termination @@ -275,7 +288,7 @@ Karpenter should terminate the now empty nodes. ```bash kubectl delete deployment inflate -kubectl logs -f -n karpenter $(kubectl get pods -n karpenter -l karpenter=controller -o name) +kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller ``` ### Manual Node Termination @@ -296,11 +309,11 @@ To avoid additional charges, remove the demo infrastructure from your AWS accoun ```bash helm uninstall karpenter --namespace karpenter -eksctl delete iamserviceaccount --cluster ${CLUSTER_NAME} --name karpenter --namespace karpenter -aws cloudformation delete-stack --stack-name Karpenter-${CLUSTER_NAME} +aws iam delete-role --role-name "${CLUSTER_NAME}-karpenter" +aws cloudformation delete-stack --stack-name "Karpenter-${CLUSTER_NAME}" aws ec2 describe-launch-templates \ | jq -r ".LaunchTemplates[].LaunchTemplateName" \ - | grep -i Karpenter-${CLUSTER_NAME} \ + | grep -i "Karpenter-${CLUSTER_NAME}" \ | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} -eksctl delete cluster --name ${CLUSTER_NAME} +eksctl delete cluster --name "${CLUSTER_NAME}" ```