Skip to content

Commit

Permalink
autoscaler e2e test
Browse files Browse the repository at this point in the history
  • Loading branch information
Yuvaraj Kakaraparthi committed Apr 15, 2023
1 parent e774ad5 commit b020fb9
Show file tree
Hide file tree
Showing 12 changed files with 991 additions and 33 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ generate-e2e-templates-main: $(KUSTOMIZE)
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-kcp-scale-in --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-kcp-scale-in.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ipv6 --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ipv6.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-single-node-cluster --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-single-node-cluster.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology-autoscaler.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-topology --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-topology.yaml
$(KUSTOMIZE) build $(DOCKER_TEMPLATES)/main/cluster-template-ignition --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/main/cluster-template-ignition.yaml

Expand Down
18 changes: 18 additions & 0 deletions api/v1beta1/common_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,24 @@ const (
// This annotation can be used to inform MachinePool status during in-progress scaling scenarios.
ReplicasManagedByAnnotation = "cluster.x-k8s.io/replicas-managed-by"

// AutoscalerMinSizeAnnotation defines the minimum node group size.
// The annotation is used by autoscaler.
// The annotation is copied from kubernetes/autoscaler.
// Ref:https://github.com/kubernetes/autoscaler/blob/d8336cca37dbfa5d1cb7b7e453bd511172d6e5e7/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go#L256-L259
// Note: With the Kubernetes autoscaler it is possible to use different annotations by configuring a different
// "Cluster API group" than "cluster.x-k8s.io" via the "CAPI_GROUP" environment variable.
// We only handle the default group in our implementation.
AutoscalerMinSizeAnnotation = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"

// AutoscalerMaxSizeAnnotation defines the maximum node group size.
// The annotations is used by the autoscaler.
// The annotation definition is copied from kubernetes/autoscaler.
// Ref:https://github.com/kubernetes/autoscaler/blob/d8336cca37dbfa5d1cb7b7e453bd511172d6e5e7/cluster-autoscaler/cloudprovider/clusterapi/clusterapi_utils.go#L264-L267
// Note: With the Kubernetes autoscaler it is possible to use different annotations by configuring a different
// "Cluster API group" than "cluster.x-k8s.io" via the "CAPI_GROUP" environment variable.
// We only handle the default group in our implementation.
AutoscalerMaxSizeAnnotation = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"

// VariableDefinitionFromInline indicates a patch or variable was defined in the `.spec` of a ClusterClass
// rather than from an external patch extension.
VariableDefinitionFromInline = "inline"
Expand Down
24 changes: 8 additions & 16 deletions api/v1beta1/machinedeployment_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,14 +267,6 @@ func (m *MachineDeployment) validate(old *MachineDeployment) error {
return apierrors.NewInvalid(GroupVersion.WithKind("MachineDeployment").GroupKind(), m.Name, allErrs)
}

// With the Kubernetes autoscaler it is possible to use different annotations by configuring a different
// "Cluster API group" than "cluster.x-k8s.io" via the "CAPI_GROUP" environment variable.
// We only handle the default group in our implementation.
const (
autoscalerMinSize = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"
autoscalerMaxSize = "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"
)

// calculateMachineDeploymentReplicas calculates the default value of the replicas field.
// The value will be calculated based on the following logic:
// * if replicas is already set on newMD, keep the current value
Expand Down Expand Up @@ -314,23 +306,23 @@ func calculateMachineDeploymentReplicas(ctx context.Context, oldMD *MachineDeplo
log := ctrl.LoggerFrom(ctx).WithValues("MachineDeployment", klog.KObj(newMD))

// If both autoscaler annotations are set, use them to calculate the default value.
minSizeString, hasMinSizeAnnotation := newMD.Annotations[autoscalerMinSize]
maxSizeString, hasMaxSizeAnnotation := newMD.Annotations[autoscalerMaxSize]
minSizeString, hasMinSizeAnnotation := newMD.Annotations[AutoscalerMinSizeAnnotation]
maxSizeString, hasMaxSizeAnnotation := newMD.Annotations[AutoscalerMaxSizeAnnotation]
if hasMinSizeAnnotation && hasMaxSizeAnnotation {
minSize, err := strconv.ParseInt(minSizeString, 10, 32)
if err != nil {
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", autoscalerMinSize)
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", AutoscalerMinSizeAnnotation)
}
maxSize, err := strconv.ParseInt(maxSizeString, 10, 32)
if err != nil {
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", autoscalerMaxSize)
return 0, errors.Wrapf(err, "failed to caculate MachineDeployment replicas value: could not parse the value of the %q annotation", AutoscalerMaxSizeAnnotation)
}

// If it's a new MachineDeployment => Use the min size.
// Note: This will result in a scale up to get into the range where autoscaler takes over.
if oldMD == nil {
if !dryRun {
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (MD is a new MD)", minSize, autoscalerMinSize))
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (MD is a new MD)", minSize, AutoscalerMinSizeAnnotation))
}
return int32(minSize), nil
}
Expand All @@ -344,21 +336,21 @@ func calculateMachineDeploymentReplicas(ctx context.Context, oldMD *MachineDeplo
// We only have this handling to be 100% safe against panics.
case oldMD.Spec.Replicas == nil:
if !dryRun {
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD didn't have replicas set)", minSize, autoscalerMinSize))
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD didn't have replicas set)", minSize, AutoscalerMinSizeAnnotation))
}
return int32(minSize), nil
// If the old MachineDeployment replicas are lower than min size => Use the min size.
// Note: This will result in a scale up to get into the range where autoscaler takes over.
case *oldMD.Spec.Replicas < int32(minSize):
if !dryRun {
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas below min size)", minSize, autoscalerMinSize))
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas below min size)", minSize, AutoscalerMinSizeAnnotation))
}
return int32(minSize), nil
// If the old MachineDeployment replicas are higher than max size => Use the max size.
// Note: This will result in a scale down to get into the range where autoscaler takes over.
case *oldMD.Spec.Replicas > int32(maxSize):
if !dryRun {
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas above max size)", maxSize, autoscalerMaxSize))
log.V(2).Info(fmt.Sprintf("Replica field has been defaulted to %d based on the %s annotation (old MD had replicas above max size)", maxSize, AutoscalerMaxSizeAnnotation))
}
return int32(maxSize), nil
// If the old MachineDeployment replicas are between min and max size => Keep the current value.
Expand Down
32 changes: 16 additions & 16 deletions api/v1beta1/machinedeployment_webhook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "3",
AutoscalerMinSizeAnnotation: "3",
},
},
},
Expand All @@ -116,7 +116,7 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMaxSize: "7",
AutoscalerMaxSizeAnnotation: "7",
},
},
},
Expand All @@ -127,8 +127,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "abc",
autoscalerMaxSize: "7",
AutoscalerMinSizeAnnotation: "abc",
AutoscalerMaxSizeAnnotation: "7",
},
},
},
Expand All @@ -139,8 +139,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "3",
autoscalerMaxSize: "abc",
AutoscalerMinSizeAnnotation: "3",
AutoscalerMaxSizeAnnotation: "abc",
},
},
},
Expand All @@ -151,8 +151,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "3",
autoscalerMaxSize: "7",
AutoscalerMinSizeAnnotation: "3",
AutoscalerMaxSizeAnnotation: "7",
},
},
},
Expand All @@ -163,8 +163,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "3",
autoscalerMaxSize: "7",
AutoscalerMinSizeAnnotation: "3",
AutoscalerMaxSizeAnnotation: "7",
},
},
},
Expand All @@ -176,8 +176,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "3",
autoscalerMaxSize: "7",
AutoscalerMinSizeAnnotation: "3",
AutoscalerMaxSizeAnnotation: "7",
},
},
},
Expand All @@ -193,8 +193,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "3",
autoscalerMaxSize: "7",
AutoscalerMinSizeAnnotation: "3",
AutoscalerMaxSizeAnnotation: "7",
},
},
},
Expand All @@ -210,8 +210,8 @@ func TestCalculateMachineDeploymentReplicas(t *testing.T) {
newMD: &MachineDeployment{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
autoscalerMinSize: "3",
autoscalerMaxSize: "7",
AutoscalerMinSizeAnnotation: "3",
AutoscalerMaxSizeAnnotation: "7",
},
},
},
Expand Down
Loading

0 comments on commit b020fb9

Please sign in to comment.