test integration with CAPI cluster-autoscaler

kubernetes-sigs · Jun 22, 2023 · d40b1b2 · d40b1b2
1 parent f49938a
commit d40b1b2
Show file tree

Hide file tree

Showing 24 changed files with 364 additions and 11 deletions.
diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml
diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml
diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml
diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml
diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml
diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml
diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml
@@ -132,6 +132,9 @@ spec:
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
 metadata:
+  annotations:
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5"
+    cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1"
   name: ${CLUSTER_NAME}-md-0
   namespace: default
 spec:

diff --git a/templates/flavors/default/machine-deployment.yaml b/templates/flavors/default/machine-deployment.yaml
@@ -3,6 +3,9 @@ apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
 metadata:
   name: "${CLUSTER_NAME}-md-0"
+  annotations:
+    "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size": "1"
+    "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size": "5"
 spec:
   clusterName: "${CLUSTER_NAME}"
   replicas: ${WORKER_MACHINE_COUNT}

diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml
diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml
diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml
diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml
diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml
diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml
diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml
diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml
diff --git a/test/e2e/azure_csidriver.go b/test/e2e/azure_csidriver.go
@@ -94,7 +94,7 @@ func AzureDiskCSISpec(ctx context.Context, inputGetter func() AzureDiskCSISpecIn
 
 	By("creating a deployment that uses pvc")
 	deploymentName := "stateful" + util.RandomString(6)
-	statefulDeployment := deploymentBuilder.Create("nginx", deploymentName, corev1.NamespaceDefault).AddPVC(pvcName)
+	statefulDeployment := deploymentBuilder.Create("nginx", deploymentName, corev1.NamespaceDefault, int32(1)).AddPVC(pvcName)
 	deployment, err := statefulDeployment.Deploy(ctx, clientset)
 	Expect(err).NotTo(HaveOccurred())
 	waitForDeploymentAvailable(ctx, deployment, clientset, specName)

diff --git a/test/e2e/azure_lb.go b/test/e2e/azure_lb.go
@@ -77,7 +77,7 @@ func AzureLBSpec(ctx context.Context, inputGetter func() AzureLBSpecInput) {
 		deploymentName = "web-windows" + util.RandomString(6)
 	}
 
-	webDeployment := deploymentBuilder.Create("httpd", deploymentName, corev1.NamespaceDefault)
+	webDeployment := deploymentBuilder.Create("httpd", deploymentName, corev1.NamespaceDefault, int32(1))
 	webDeployment.AddContainerPort("http", "http", 80, corev1.ProtocolTCP)
 
 	if input.Windows {

diff --git a/test/e2e/azure_machinepool_drain.go b/test/e2e/azure_machinepool_drain.go
@@ -274,7 +274,7 @@ func deployHTTPService(ctx context.Context, clientset *kubernetes.Clientset, isW
 
 			return "web" + util.RandomString(6)
 		}()
-		webDeploymentBuilder = deployments.Create("httpd", deploymentName, corev1.NamespaceDefault)
+		webDeploymentBuilder = deployments.Create("httpd", deploymentName, corev1.NamespaceDefault, int32(1))
 		servicesClient       = clientset.CoreV1().Services(corev1.NamespaceDefault)
 		ports                = []corev1.ServicePort{
 			{

diff --git a/test/e2e/azure_net_pol.go b/test/e2e/azure_net_pol.go
@@ -96,15 +96,15 @@ func AzureNetPolSpec(ctx context.Context, inputGetter func() AzureNetPolSpecInpu
 	// Front end Prod
 	frontendProdDeploymentName := fmt.Sprintf("frontend-prod-%v", randInt)
 	Log("starting to create frontend-prod deployments")
-	frontEndProd := deploymentBuilder.Create("library/nginx:1.21.1", frontendProdDeploymentName, namespaceProd.GetName())
+	frontEndProd := deploymentBuilder.Create("library/nginx:1.21.1", frontendProdDeploymentName, namespaceProd.GetName(), int32(1))
 	frontEndProd.AddLabels(frontendLabels)
 	frontendProdDeployment, err := frontEndProd.Deploy(ctx, clientset)
 	Expect(err).NotTo(HaveOccurred())
 
 	// Front end Dev
 	frontendDevDeploymentName := fmt.Sprintf("frontend-dev-%v", randInt+100000)
 	Log("starting to create frontend-dev deployments")
-	frontEndDev := deploymentBuilder.Create("library/nginx:1.21.1", frontendDevDeploymentName, namespaceDev.GetName())
+	frontEndDev := deploymentBuilder.Create("library/nginx:1.21.1", frontendDevDeploymentName, namespaceDev.GetName(), int32(1))
 	frontEndDev.AddLabels(frontendLabels)
 	frontendDevDeployment, err := frontEndDev.Deploy(ctx, clientset)
 	Expect(err).NotTo(HaveOccurred())
@@ -113,7 +113,7 @@ func AzureNetPolSpec(ctx context.Context, inputGetter func() AzureNetPolSpecInpu
 	backendDeploymentName := fmt.Sprintf("backend-%v", randInt+200000)
 	backendLabels := map[string]string{"app": "webapp", "role": "backend"}
 	Log("starting to create backend deployments")
-	backendDev := deploymentBuilder.Create("library/nginx:1.21.1", backendDeploymentName, namespaceDev.GetName())
+	backendDev := deploymentBuilder.Create("library/nginx:1.21.1", backendDeploymentName, namespaceDev.GetName(), int32(1))
 	backendDev.AddLabels(backendLabels)
 	backendDeployment, err := backendDev.Deploy(ctx, clientset)
 	Expect(err).NotTo(HaveOccurred())
@@ -122,7 +122,7 @@ func AzureNetPolSpec(ctx context.Context, inputGetter func() AzureNetPolSpecInpu
 	nwpolicyDeploymentName := fmt.Sprintf("network-policy-%v", randInt+300000)
 	nwpolicyLabels := map[string]string{"app": "webapp", "role": "any"}
 	Log("starting to create network-policy deployments")
-	nwpolicy := deploymentBuilder.Create("library/nginx:1.21.1", nwpolicyDeploymentName, namespaceDev.GetName())
+	nwpolicy := deploymentBuilder.Create("library/nginx:1.21.1", nwpolicyDeploymentName, namespaceDev.GetName(), int32(1))
 	nwpolicy.AddLabels(nwpolicyLabels)
 	nwpolicyDeployment, err := nwpolicy.Deploy(ctx, clientset)
 	Expect(err).NotTo(HaveOccurred())

diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go
@@ -182,6 +182,13 @@ var _ = Describe("Workload cluster creation", func() {
 								ClusterName:           clusterName,
 							}
 						})
+						InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+							return ClusterAutoscalerInstallInput{
+								BootstrapClusterProxy: bootstrapClusterProxy,
+								Namespace:             namespace,
+								ClusterName:           clusterName,
+							}
+						})
 					}),
 				), result)
 
@@ -232,9 +239,26 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
+			By("Scaling out via cluster-autoscaler", func() {
+				AzureAutoscalerSpec(ctx, func() AzureAutoscalerSpecInput {
+					return AzureAutoscalerSpecInput{
+						BootstrapClusterProxy: bootstrapClusterProxy,
+						Namespace:             namespace,
+						ClusterName:           clusterName,
+					}
+				})
+			})
+
 			By("Verifying expected VM extensions are present on the node", func() {
 				AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput {
 					return AzureVMExtensionsSpecInput{
@@ -294,7 +318,7 @@ var _ = Describe("Workload cluster creation", func() {
 		})
 	})
 
-	When("Creating a highly available cluster with Azure CNI v1 [REQUIRED]", Label("Azure CNI v1"), func() {
+	When("Creating a cluster with Azure CNI v1 [REQUIRED]", Label("Azure CNI v1"), func() {
 		It("can create 3 control-plane nodes and 2 Linux worker nodes", func() {
 			clusterName = getClusterName(clusterNamePrefix, "azcni-v1")
 
@@ -377,6 +401,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
@@ -415,6 +446,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
@@ -475,6 +513,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
@@ -561,6 +606,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
@@ -669,6 +721,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
@@ -847,6 +906,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
@@ -921,6 +987,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)
 
@@ -965,6 +1038,13 @@ var _ = Describe("Workload cluster creation", func() {
 							ClusterName:           clusterName,
 						}
 					})
+					InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput {
+						return ClusterAutoscalerInstallInput{
+							BootstrapClusterProxy: bootstrapClusterProxy,
+							Namespace:             namespace,
+							ClusterName:           clusterName,
+						}
+					})
 				}),
 			), result)