From 1c94cfdefda412c586dc874a4b3249d74f807bc3 Mon Sep 17 00:00:00 2001 From: edibble21 <85638465+edibble21@users.noreply.github.com> Date: Wed, 4 Dec 2024 10:45:28 -0800 Subject: [PATCH 1/5] =?UTF-8?q?Docs:=20Added=20note=20to=20migration=20gui?= =?UTF-8?q?de=20for=20expiration=20causing=20more=20pods=20in=20sch?= =?UTF-8?q?=E2=80=A6=20(#7474)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- website/content/en/v1.0/upgrading/v1-migration.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/website/content/en/v1.0/upgrading/v1-migration.md b/website/content/en/v1.0/upgrading/v1-migration.md index 7b242c404fd3..eb502eb36dab 100644 --- a/website/content/en/v1.0/upgrading/v1-migration.md +++ b/website/content/en/v1.0/upgrading/v1-migration.md @@ -21,6 +21,10 @@ Use the [compatibility matrix]({{}}) t Karpenter `v1.0` is a major release and contains a number of breaking changes. The following section will highlight some of the major breaking changes, but you should review the full [changelog]({{}}) before proceeding with the upgrade. +#### Forceful Expiration + +Previously, Karpenter would only begin draining an expired node after ensuring graceful disruption was possible and replacement capacity was provisioned. Now, NodeClaim expiration is forceful. Once a NodeClaim expires, Karpenter will immediately begin draining the corresponding Node without first provisioning a replacement Node. This forceful approach means that all expired NodeClaims that were previously blocked from disruption (due to Pod Disruption Budgets (PDBs), do-not-disrupt pods, or other constraints) will be expired. The scheduler will then attempt to provision new Nodes for the disrupted workloads as they are draining. This behavior may lead to an increased number of pods in the "Pending" state while replacement capacity is being provisioned. Before upgrading, users should check for expired NodeClaims and resolve any disruption blockers to ensure a smooth upgrade process. + #### Deprecated Annotations, Labels, and Tags Removed The following annotations, labels, and tags have been removed in `v1.0.0`: From 8007f15848686270cfe4c9fc4874d71ba9aadcaf Mon Sep 17 00:00:00 2001 From: andyblog Date: Thu, 5 Dec 2024 06:34:07 +0800 Subject: [PATCH 2/5] docs: fix karpenter service namespace (#7481) --- hack/docs/metrics_gen/main.go | 2 +- website/content/en/docs/reference/metrics.md | 2 +- website/content/en/preview/reference/metrics.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/docs/metrics_gen/main.go b/hack/docs/metrics_gen/main.go index 63fd06391c5f..018dc5c24fc6 100644 --- a/hack/docs/metrics_gen/main.go +++ b/hack/docs/metrics_gen/main.go @@ -111,7 +111,7 @@ description: > `) fmt.Fprintf(f, "\n") fmt.Fprintf(f, "Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. "+ - "These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings)\n") + "These metrics are available by default at `karpenter.kube-system.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings)\n") previousSubsystem := "" for _, metric := range allMetrics { diff --git a/website/content/en/docs/reference/metrics.md b/website/content/en/docs/reference/metrics.md index 1f1d16c67a31..53d0cfe0e555 100644 --- a/website/content/en/docs/reference/metrics.md +++ b/website/content/en/docs/reference/metrics.md @@ -7,7 +7,7 @@ description: > Inspect Karpenter Metrics --- -Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) +Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.kube-system.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) ### `karpenter_ignored_pod_count` Number of pods ignored during scheduling by Karpenter - Stability Level: ALPHA diff --git a/website/content/en/preview/reference/metrics.md b/website/content/en/preview/reference/metrics.md index 1f1d16c67a31..53d0cfe0e555 100644 --- a/website/content/en/preview/reference/metrics.md +++ b/website/content/en/preview/reference/metrics.md @@ -7,7 +7,7 @@ description: > Inspect Karpenter Metrics --- -Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) +Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.kube-system.svc.cluster.local:8080/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) ### `karpenter_ignored_pod_count` Number of pods ignored during scheduling by Karpenter - Stability Level: ALPHA From 51eafa6e39036dacda23adbf4269c8e701822e18 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Wed, 4 Dec 2024 14:57:56 -0800 Subject: [PATCH 3/5] chore: Bump `sigs.k8s.io/karpenter` to latest (#7484) --- go.mod | 7 +++---- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 35a9973acafb..67ab9a1bccbb 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881 github.com/aws/smithy-go v1.22.1 github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 - github.com/awslabs/operatorpkg v0.0.0-20241130193813-da0c28954330 + github.com/awslabs/operatorpkg v0.0.0-20241204220700-cc7010c0de7a github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.1 @@ -42,8 +42,8 @@ require ( k8s.io/client-go v0.31.3 k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 - sigs.k8s.io/controller-runtime v0.19.2 - sigs.k8s.io/karpenter v1.1.1-0.20241130200058-b1b45fc9218b + sigs.k8s.io/controller-runtime v0.19.3 + sigs.k8s.io/karpenter v1.1.1-0.20241204224505-787a08f73c15 sigs.k8s.io/yaml v1.4.0 ) @@ -65,7 +65,6 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect diff --git a/go.sum b/go.sum index 1bbd6415540e..2de38e182a69 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 h1:8yRBVsjGmI7qQsPWtIrbWP+XfwHO9Wq7gdLVzjqiZFs= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647/go.mod h1:9NafTAUHL0FlMeL6Cu5PXnMZ1q/LnC9X2emLXHsVbM8= -github.com/awslabs/operatorpkg v0.0.0-20241130193813-da0c28954330 h1:tkG3SqBneysUkSBIAV6Am8Bu8Aux3OabsLNYscFuqUU= -github.com/awslabs/operatorpkg v0.0.0-20241130193813-da0c28954330/go.mod h1:/d+XFCUzpwskkp4FyeftGMESTG+BitZ+OLugaCrKxuQ= +github.com/awslabs/operatorpkg v0.0.0-20241204220700-cc7010c0de7a h1:f7FeAHGBChhxp29xi5Up+hT4pot62w0PoQaFKHY4eJc= +github.com/awslabs/operatorpkg v0.0.0-20241204220700-cc7010c0de7a/go.mod h1:TTs6HGuqmgdNyNlbdv29v1OoON+kQKVPojZgJaJVtNk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -74,8 +74,6 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -314,12 +312,12 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/controller-runtime v0.19.2 h1:3sPrF58XQEPzbE8T81TN6selQIMGbtYwuaJ6eDssDF8= -sigs.k8s.io/controller-runtime v0.19.2/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4= +sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8bX1sPw= +sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v1.1.1-0.20241130200058-b1b45fc9218b h1:u+LZc5vsQdCWD8WvdrKWBOWbfgCfTKZ3un5Zx9Hj/GA= -sigs.k8s.io/karpenter v1.1.1-0.20241130200058-b1b45fc9218b/go.mod h1:cz71ZPhOxoXpdt5InLs6z1WUba3lMwEnea7olWZgWcY= +sigs.k8s.io/karpenter v1.1.1-0.20241204224505-787a08f73c15 h1:7wuMSt9RzkZTCRvu4rF5q3IL3w4ZXBCggdG1YK3jnlE= +sigs.k8s.io/karpenter v1.1.1-0.20241204224505-787a08f73c15/go.mod h1:aM/nBvDb5P4aBP0YfKBY+QWbilCJYuLXAcL6eV3MP6s= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= From 4ad80a56d8d623057fd4049cb08ef41e8fb71f4c Mon Sep 17 00:00:00 2001 From: Robin Ketelbuters Date: Thu, 5 Dec 2024 19:56:49 +0100 Subject: [PATCH 4/5] feat: Enable setting schedulerName in Helm chart (#7456) Co-authored-by: Robin Ketelbuters --- charts/karpenter/templates/deployment.yaml | 3 +++ charts/karpenter/values.yaml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml index c48718497aeb..f43822270dd6 100644 --- a/charts/karpenter/templates/deployment.yaml +++ b/charts/karpenter/templates/deployment.yaml @@ -56,6 +56,9 @@ spec: {{- if .Values.hostNetwork }} hostNetwork: true {{- end }} + {{- with .Values.schedulerName }} + schedulerName: {{ . | quote }} + {{- end }} containers: - name: controller securityContext: diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index e3d182240d59..c560190c3c4c 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -59,6 +59,8 @@ terminationGracePeriodSeconds: # -- Bind the pod to the host network. # This is required when using a custom CNI. hostNetwork: false +# -- Specify which Kubernetes scheduler should dispatch the pod. +schedulerName: default-scheduler # -- Configure the DNS Policy for the pod dnsPolicy: ClusterFirst # -- Configure DNS Config for the pod From 66df0efeb6ad45f5ac9fa007f069d1632f93a4de Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Thu, 5 Dec 2024 11:08:39 -0800 Subject: [PATCH 5/5] chore: Bump `sigs.k8s.io/karpenter` to latest (#7489) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 67ab9a1bccbb..e31ecca50052 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881 github.com/aws/smithy-go v1.22.1 github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 - github.com/awslabs/operatorpkg v0.0.0-20241204220700-cc7010c0de7a + github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115 github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 github.com/jonathan-innis/aws-sdk-go-prometheus v0.1.1 @@ -43,7 +43,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.19.3 - sigs.k8s.io/karpenter v1.1.1-0.20241204224505-787a08f73c15 + sigs.k8s.io/karpenter v1.1.1-0.20241205183200-554660337d6e sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 2de38e182a69..d65a2965e048 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 h1:8yRBVsjGmI7qQsPWtIrbWP+XfwHO9Wq7gdLVzjqiZFs= github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647/go.mod h1:9NafTAUHL0FlMeL6Cu5PXnMZ1q/LnC9X2emLXHsVbM8= -github.com/awslabs/operatorpkg v0.0.0-20241204220700-cc7010c0de7a h1:f7FeAHGBChhxp29xi5Up+hT4pot62w0PoQaFKHY4eJc= -github.com/awslabs/operatorpkg v0.0.0-20241204220700-cc7010c0de7a/go.mod h1:TTs6HGuqmgdNyNlbdv29v1OoON+kQKVPojZgJaJVtNk= +github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115 h1:9nhjY3dzCpEmhpQ0vMlhB7wqucAiftLjAIEQu8uT2J4= +github.com/awslabs/operatorpkg v0.0.0-20241205163410-0fff9f28d115/go.mod h1:TTs6HGuqmgdNyNlbdv29v1OoON+kQKVPojZgJaJVtNk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -316,8 +316,8 @@ sigs.k8s.io/controller-runtime v0.19.3 h1:XO2GvC9OPftRst6xWCpTgBZO04S2cbp0Qqkj8b sigs.k8s.io/controller-runtime v0.19.3/go.mod h1:j4j87DqtsThvwTv5/Tc5NFRyyF/RF0ip4+62tbTSIUM= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v1.1.1-0.20241204224505-787a08f73c15 h1:7wuMSt9RzkZTCRvu4rF5q3IL3w4ZXBCggdG1YK3jnlE= -sigs.k8s.io/karpenter v1.1.1-0.20241204224505-787a08f73c15/go.mod h1:aM/nBvDb5P4aBP0YfKBY+QWbilCJYuLXAcL6eV3MP6s= +sigs.k8s.io/karpenter v1.1.1-0.20241205183200-554660337d6e h1:30Jg5ZIThgpRm7InPv+5w3AX8AOg3AWaJklesL7qpgM= +sigs.k8s.io/karpenter v1.1.1-0.20241205183200-554660337d6e/go.mod h1:NQouOJNK6s1d4EIKa5cY7nAV3IG74qZ6gPzHBeCZNPw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=