feat(cluster-autoscaler) Implement emitPerNodegroupMetrics parameter

hakman · Aug 10, 2024 · cdc645a · cdc645a
1 parent 55bdf20
commit cdc645a
Show file tree

Hide file tree

Showing 37 changed files with 64 additions and 8 deletions.
diff --git a/docs/addons.md b/docs/addons.md
@@ -71,6 +71,7 @@ spec:
     enabled: true
     expander: least-waste
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     awsUseStaticInstanceList: false
     scaleDownUtilizationThreshold: 0.5
     skipNodesWithCustomControllerPods: true

diff --git a/docs/releases/1.29-NOTES.md b/docs/releases/1.29-NOTES.md
@@ -2,6 +2,8 @@
 
 # Significant changes
 
+* Add support to --emit-per-nodegroup-metrics on the cluster autoscaler addon (emitPerNodegroupMetrics) [PR#16693](https://github.com/kubernetes/kops/pull/16693/)
+
 ## Deferred deletion / pruning phase
 
 Some infrastructure changes are potentially disruptive to the continued

diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -769,6 +769,11 @@ spec:
                       CustomPriorityExpanderConfig overides the priority-expander ConfigMap with the provided configuration. Any InstanceGroup configuration will be ignored if this is set.
                       This could be useful in order to use regex on priorities configuration
                     type: object
+                  emitPerNodegroupMetrics:
+                    description: |-
+                      EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+                      Default: false
+                    type: boolean
                   enabled:
                     description: |-
                       Enabled enables the cluster autoscaler.

diff --git a/pkg/apis/kops/componentconfig.go b/pkg/apis/kops/componentconfig.go
@@ -1088,6 +1088,9 @@ type ClusterAutoscalerConfig struct {
 	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+	// Default: false
+	EmitPerNodegroupMetrics *bool `json:"emitPerNodegroupMetrics,omitempty"`
 	// AWSUseStaticInstanceList makes cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`

diff --git a/pkg/apis/kops/v1alpha2/componentconfig.go b/pkg/apis/kops/v1alpha2/componentconfig.go
@@ -1151,6 +1151,9 @@ type ClusterAutoscalerConfig struct {
 	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+	// Default: false
+	EmitPerNodegroupMetrics *bool `json:"emitPerNodegroupMetrics,omitempty"`
 	// AWSUseStaticInstanceList makes the cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`

diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
diff --git a/pkg/apis/kops/v1alpha3/componentconfig.go b/pkg/apis/kops/v1alpha3/componentconfig.go
@@ -1080,6 +1080,9 @@ type ClusterAutoscalerConfig struct {
 	// BalanceSimilarNodeGroups makes the cluster autoscaler treat similar node groups as one.
 	// Default: false
 	BalanceSimilarNodeGroups *bool `json:"balanceSimilarNodeGroups,omitempty"`
+	// EmitPerNodegroupMetrics If true, publishes the node groups min and max metrics count set on the cluster autoscaler.
+	// Default: false
+	EmitPerNodegroupMetrics *bool `json:"emitPerNodegroupMetrics,omitempty"`
 	// AWSUseStaticInstanceList makes the cluster autoscaler to use statically defined set of AWS EC2 Instance List.
 	// Default: false
 	AWSUseStaticInstanceList *bool `json:"awsUseStaticInstanceList,omitempty"`

diff --git a/pkg/apis/kops/v1alpha3/zz_generated.conversion.go b/pkg/apis/kops/v1alpha3/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha3/zz_generated.deepcopy.go
diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go
diff --git a/pkg/model/components/clusterautoscaler.go b/pkg/model/components/clusterautoscaler.go
@@ -83,6 +83,9 @@ func (b *ClusterAutoscalerOptionsBuilder) BuildOptions(o interface{}) error {
 	if cas.BalanceSimilarNodeGroups == nil {
 		cas.BalanceSimilarNodeGroups = fi.PtrTo(false)
 	}
+	if cas.EmitPerNodegroupMetrics == nil {
+		cas.EmitPerNodegroupMetrics = fi.PtrTo(false)
+	}
 	if cas.AWSUseStaticInstanceList == nil {
 		cas.AWSUseStaticInstanceList = fi.PtrTo(false)
 	}

diff --git a/...stom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-bootstrap_content b/...stom/data/aws_s3_object_cas-priority-expander-custom.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 963767d82e85200787e166f3cc6e7b3b9e60d2383c9fef71a562d9e4eedd7086
+    manifestHash: e4817b8481df3f556c19999b552e60c6305ce8063481c6ec5d987f225aa686ad
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...rity-expander-custom.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...rity-expander-custom.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -350,6 +350,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=priority

diff --git a/...ter-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content b/...ter-autoscaler-priority-expander-custom/data/aws_s3_object_cluster-completed.spec_content
@@ -34,6 +34,7 @@ spec:
       - .*low.*
       "100":
       - .*high.*
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: priority
     ignoreDaemonSetsUtilization: false

diff --git a/...ty-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-bootstrap_content b/...ty-expander/data/aws_s3_object_cas-priority-expander.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: d28605db2e6ade6fee371d462c0e9893871ea73a35ad598fa616a2657290d319
+    manifestHash: 8c697d6e92263502eeeb637b47669c5fbb3ba11cd82c90a653cddb42060e7fe5
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...as-priority-expander.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...as-priority-expander.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -350,6 +350,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=priority

diff --git a/...er/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content b/...er/cluster-autoscaler-priority-expander/data/aws_s3_object_cluster-completed.spec_content
@@ -27,6 +27,7 @@ spec:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
     createPriorityExpanderConfig: true
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: priority
     ignoreDaemonSetsUtilization: false

diff --git a/...ion/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content b/...ion/update_cluster/many-addons-ccm-irsa/data/aws_s3_object_cluster-completed.spec_content
@@ -30,6 +30,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

diff --git a/...ster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...ster/many-addons-ccm-irsa/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 10a6f07c81b4244162cf161255fd1aae420c1eada3e262aabe4f6930b2f5a65f
+    manifestHash: 3764bb2698fcd6ad15853cf437c1664e7f46acfee77fdc3c686f1aa000ab4006
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -321,6 +321,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

diff --git a/...n/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content b/...n/update_cluster/many-addons-ccm-irsa25/data/aws_s3_object_cluster-completed.spec_content
@@ -30,6 +30,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

diff --git a/...er/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...er/many-addons-ccm-irsa25/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -48,7 +48,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 114e13595bf4571c767f786144e013bc2b100fa35d23c83ecc8af940c4455cf4
+    manifestHash: c03c6c1e8806b7e572506d2018bdbb9c314f5bc207473d85e95e7308af4d5193
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -321,6 +321,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

diff --git a/...n/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content b/...n/update_cluster/many-addons-ccm-irsa26/data/aws_s3_object_cluster-completed.spec_content
@@ -30,6 +30,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

diff --git a/...er/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...er/many-addons-ccm-irsa26/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -49,7 +49,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 10a6f07c81b4244162cf161255fd1aae420c1eada3e262aabe4f6930b2f5a65f
+    manifestHash: 3764bb2698fcd6ad15853cf437c1664e7f46acfee77fdc3c686f1aa000ab4006
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -321,6 +321,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

diff --git a/...egration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content b/...egration/update_cluster/many-addons-ccm/data/aws_s3_object_cluster-completed.spec_content
@@ -31,6 +31,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

diff --git a/...e_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...e_cluster/many-addons-ccm/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 39af22339588beb7c2fd4b2b5c9e060a6c84a9fc2d8efac9b078a9ffff64753f
+    manifestHash: 45d2bf5133e21bd182463572db9f524648214e3123c8273609d0112cd9ca925a
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -327,6 +327,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

diff --git a/...egration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content b/...egration/update_cluster/many-addons-gce/data/aws_s3_object_cluster-completed.spec_content
@@ -31,6 +31,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

diff --git a/...e_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content b/...e_cluster/many-addons-gce/data/aws_s3_object_minimal.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 83138df304119f6137709766b9f76c23743f5cfc2a0d9400fb6e51444e15fde4
+    manifestHash: a60534cf6f674d86d17a32490fef182ebf5d821523402ea8cbb1d589eaec5031
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...ws_s3_object_minimal.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -327,6 +327,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=gce
         - --expander=random
         - --nodes=1:1:https://www.googleapis.com/compute/v1/projects/testproject/zones/us-test1-a/instanceGroups/a-nodes-minimal-example-com

diff --git a/.../integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content b/.../integration/update_cluster/many-addons/data/aws_s3_object_cluster-completed.spec_content
@@ -31,6 +31,7 @@ spec:
   clusterAutoscaler:
     awsUseStaticInstanceList: false
     balanceSimilarNodeGroups: false
+    emitPerNodegroupMetrics: false
     enabled: true
     expander: random
     ignoreDaemonSetsUtilization: false

diff --git a/...e_cluster/many-addons/data/aws_s3_object_many-addons.example.com-addons-bootstrap_content b/...e_cluster/many-addons/data/aws_s3_object_many-addons.example.com-addons-bootstrap_content
@@ -41,7 +41,7 @@ spec:
     version: 9.99.0
   - id: k8s-1.15
     manifest: cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml
-    manifestHash: 48d9a078fde2ad3617492f8bae23edac27fdf0d9b6102d298623b8ccf010a708
+    manifestHash: 44be0298541ea2f643250d74df219ea0846304d14bbb49529773830e028106e9
     name: cluster-autoscaler.addons.k8s.io
     selector:
       k8s-addon: cluster-autoscaler.addons.k8s.io

diff --git a/...3_object_many-addons.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content b/...3_object_many-addons.example.com-addons-cluster-autoscaler.addons.k8s.io-k8s-1.15_content
@@ -328,6 +328,7 @@ spec:
       - command:
         - ./cluster-autoscaler
         - --balance-similar-node-groups=false
+        - --emit-per-nodegroup-metrics=false
         - --cloud-provider=aws
         - --aws-use-static-instance-list=false
         - --expander=random

diff --git a/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template b/upup/models/cloudup/resources/addons/cluster-autoscaler.addons.k8s.io/k8s-1.15.yaml.template
@@ -333,6 +333,7 @@ spec:
           command:
             - ./cluster-autoscaler
             - --balance-similar-node-groups={{ .BalanceSimilarNodeGroups }}
+            - --emit-per-nodegroup-metrics={{ .EmitPerNodegroupMetrics }}
             - --cloud-provider={{ GetCloudProvider }}
             {{ if (eq GetCloudProvider "aws") }}
             - --aws-use-static-instance-list={{ .AWSUseStaticInstanceList }}