leave a buffer of underutilized nodes when scaling down

cluster-autoscaler/config/autoscaling_options.go

@@ -149,6 +149,8 @@ type AutoscalingOptions struct {
 	// The formula to calculate additional candidates number is following:
 	// max(#nodes * ScaleDownCandidatesPoolRatio, ScaleDownCandidatesPoolMinCount)
 	ScaleDownCandidatesPoolMinCount int
+	// ScaleDownBufferRatio Ratio of empty or underutilized nodes to leave as capacity buffer per nodegroup
+	ScaleDownBufferRatio float64
 	// ScaleDownSimulationTimeout defines the maximum time that can be
 	// spent on scale down simulation.
 	ScaleDownSimulationTimeout time.Duration

cluster-autoscaler/core/scaledown/unneeded/nodes.go

@@ -17,6 +17,7 @@ limitations under the License.
 package unneeded
 
 import (
+	"math"
 	"reflect"
 	"time"
 
@@ -121,18 +122,19 @@ func (n *Nodes) RemovableAt(context *context.AutoscalingContext, ts time.Time, r
 	nodeGroupSize := utils.GetNodeGroupSizeMap(context.CloudProvider)
 	resourcesLeftCopy := resourcesLeft.DeepCopy()
 	emptyNodes, drainNodes := n.splitEmptyAndNonEmptyNodes()
+	nodeGroupToBeRemovedCounter := make(map[string]int)
 
 	for nodeName, v := range emptyNodes {
 		klog.V(2).Infof("%s was unneeded for %s", nodeName, ts.Sub(v.since).String())
-		if r := n.unremovableReason(context, v, ts, nodeGroupSize, resourcesLeftCopy, resourcesWithLimits, as); r != simulator.NoReason {
+		if r := n.unremovableReason(context, v, ts, nodeGroupSize, resourcesLeftCopy, resourcesWithLimits, as, nodeGroupToBeRemovedCounter); r != simulator.NoReason {
 			unremovable = append(unremovable, &simulator.UnremovableNode{Node: v.ntbr.Node, Reason: r})
 			continue
 		}
 		empty = append(empty, v.ntbr)
 	}
 	for nodeName, v := range drainNodes {
 		klog.V(2).Infof("%s was unneeded for %s", nodeName, ts.Sub(v.since).String())
-		if r := n.unremovableReason(context, v, ts, nodeGroupSize, resourcesLeftCopy, resourcesWithLimits, as); r != simulator.NoReason {
+		if r := n.unremovableReason(context, v, ts, nodeGroupSize, resourcesLeftCopy, resourcesWithLimits, as, nodeGroupToBeRemovedCounter); r != simulator.NoReason {
 			unremovable = append(unremovable, &simulator.UnremovableNode{Node: v.ntbr.Node, Reason: r})
 			continue
 		}
@@ -141,7 +143,7 @@ func (n *Nodes) RemovableAt(context *context.AutoscalingContext, ts time.Time, r
 	return
 }
 
-func (n *Nodes) unremovableReason(context *context.AutoscalingContext, v *node, ts time.Time, nodeGroupSize map[string]int, resourcesLeft resource.Limits, resourcesWithLimits []string, as scaledown.ActuationStatus) simulator.UnremovableReason {
+func (n *Nodes) unremovableReason(context *context.AutoscalingContext, v *node, ts time.Time, nodeGroupSize map[string]int, resourcesLeft resource.Limits, resourcesWithLimits []string, as scaledown.ActuationStatus, nodeGroupToBeRemovedCounter map[string]int) simulator.UnremovableReason {
 	node := v.ntbr.Node
 	// Check if node is marked with no scale down annotation.
 	if eligibility.HasNoScaleDownAnnotation(node) {
@@ -208,6 +210,17 @@ func (n *Nodes) unremovableReason(context *context.AutoscalingContext, v *node,
 		return simulator.MinimalResourceLimitExceeded
 	}
 
+	if context.ScaleDownBufferRatio != 0.0 {
+		// always leave a buffer of nodes to not scale down
+		// this always needs to be the last check since it assumes everything else is scaled down
+		buffer := int(math.Round(float64(nodeGroupSize[nodeGroup.Id()]) * context.ScaleDownBufferRatio))
+		if nodeGroupToBeRemovedCounter[nodeGroup.Id()] <= buffer {
+			klog.V(4).Infof("Skipping %s - buffer %v not reached for %v", node.Name, buffer, nodeGroup.Id())
+			nodeGroupToBeRemovedCounter[nodeGroup.Id()]++ // so next node might pass ...
+			return simulator.ScaleDownDisabledAnnotation  // TODO: make our own reason and see why that is needed
+		}
+	}
+
 	nodeGroupSize[nodeGroup.Id()]--
 	return simulator.NoReason
 }

cluster-autoscaler/main.go

@@ -132,6 +132,8 @@ var (
 			"for scale down when some candidates from previous iteration are no longer valid."+
 			"When calculating the pool size for additional candidates we take"+
 			"max(#nodes * scale-down-candidates-pool-ratio, scale-down-candidates-pool-min-count).")
+	scaleDownBufferRatio = flag.Float64("scale-down-buffer-ratio", 0.1,
+		"Ratio of empty or underutilized nodes to leave as capacity buffer per nodegroup")
 	nodeDeletionDelayTimeout    = flag.Duration("node-deletion-delay-timeout", 2*time.Minute, "Maximum time CA waits for removing delay-deletion.cluster-autoscaler.kubernetes.io/ annotations before deleting the node.")
 	nodeDeletionBatcherInterval = flag.Duration("node-deletion-batcher-interval", 0*time.Second, "How long CA ScaleDown gather nodes to delete them in batch.")
 	scanInterval                = flag.Duration("scan-interval", 10*time.Second, "How often cluster is reevaluated for scale up or down")
@@ -287,6 +289,7 @@ func createAutoscalingOptions() config.AutoscalingOptions {
 		ScaleDownNonEmptyCandidatesCount:   *scaleDownNonEmptyCandidatesCount,
 		ScaleDownCandidatesPoolRatio:       *scaleDownCandidatesPoolRatio,
 		ScaleDownCandidatesPoolMinCount:    *scaleDownCandidatesPoolMinCount,
+		ScaleDownBufferRatio:               *scaleDownBufferRatio,
 		WriteStatusConfigMap:               *writeStatusConfigMapFlag,
 		StatusConfigMapName:                *statusConfigMapName,
 		BalanceSimilarNodeGroups:           *balanceSimilarNodeGroupsFlag,