From d40b1b28c214ec79a5193cd5bba15599f7b53bad Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 9 Jun 2023 17:37:47 -0700 Subject: [PATCH] test integration with CAPI cluster-autoscaler --- templates/cluster-template-azure-bastion.yaml | 3 + templates/cluster-template-azure-cni-v1.yaml | 3 + templates/cluster-template-edgezone.yaml | 3 + templates/cluster-template-ephemeral.yaml | 3 + templates/cluster-template-private.yaml | 3 + templates/cluster-template-windows.yaml | 3 + templates/cluster-template.yaml | 3 + .../flavors/default/machine-deployment.yaml | 3 + .../cluster-template-prow-azure-cni-v1.yaml | 3 + .../ci/cluster-template-prow-ci-version.yaml | 3 + .../ci/cluster-template-prow-custom-vnet.yaml | 3 + .../ci/cluster-template-prow-edgezone.yaml | 3 + ...r-template-prow-intree-cloud-provider.yaml | 3 + .../ci/cluster-template-prow-private.yaml | 3 + templates/test/ci/cluster-template-prow.yaml | 3 + .../dev/cluster-template-custom-builds.yaml | 3 + test/e2e/azure_csidriver.go | 2 +- test/e2e/azure_lb.go | 2 +- test/e2e/azure_machinepool_drain.go | 2 +- test/e2e/azure_net_pol.go | 8 +- test/e2e/azure_test.go | 82 ++++++- test/e2e/cluster_autoscaler.go | 211 ++++++++++++++++++ test/e2e/config/azure-dev.yaml | 3 +- test/e2e/kubernetes/deployment/deployment.go | 17 +- 24 files changed, 364 insertions(+), 11 deletions(-) create mode 100644 test/e2e/cluster_autoscaler.go diff --git a/templates/cluster-template-azure-bastion.yaml b/templates/cluster-template-azure-bastion.yaml index 98ffb2f540cc..937073141fa8 100644 --- a/templates/cluster-template-azure-bastion.yaml +++ b/templates/cluster-template-azure-bastion.yaml @@ -134,6 +134,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/cluster-template-azure-cni-v1.yaml b/templates/cluster-template-azure-cni-v1.yaml index 4de37fdcc5c7..c290ec223026 100644 --- a/templates/cluster-template-azure-cni-v1.yaml +++ b/templates/cluster-template-azure-cni-v1.yaml @@ -137,6 +137,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/cluster-template-edgezone.yaml b/templates/cluster-template-edgezone.yaml index 54b931774966..8727044f0e3e 100644 --- a/templates/cluster-template-edgezone.yaml +++ b/templates/cluster-template-edgezone.yaml @@ -135,6 +135,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/cluster-template-ephemeral.yaml b/templates/cluster-template-ephemeral.yaml index 55ec76cf7012..79250893c45b 100644 --- a/templates/cluster-template-ephemeral.yaml +++ b/templates/cluster-template-ephemeral.yaml @@ -135,6 +135,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/cluster-template-private.yaml b/templates/cluster-template-private.yaml index 252b03eab1ab..502375f261d4 100644 --- a/templates/cluster-template-private.yaml +++ b/templates/cluster-template-private.yaml @@ -146,6 +146,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/cluster-template-windows.yaml b/templates/cluster-template-windows.yaml index c84d2a1b6afb..a8e3f532084e 100644 --- a/templates/cluster-template-windows.yaml +++ b/templates/cluster-template-windows.yaml @@ -136,6 +136,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 270dd7b37587..2f1a74e4f2f4 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -132,6 +132,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/flavors/default/machine-deployment.yaml b/templates/flavors/default/machine-deployment.yaml index f63d224948aa..a2c30e02dce6 100644 --- a/templates/flavors/default/machine-deployment.yaml +++ b/templates/flavors/default/machine-deployment.yaml @@ -3,6 +3,9 @@ apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: name: "${CLUSTER_NAME}-md-0" + annotations: + "cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size": "1" + "cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size": "5" spec: clusterName: "${CLUSTER_NAME}" replicas: ${WORKER_MACHINE_COUNT} diff --git a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml index b6f02f0153cf..8d58e475d77d 100644 --- a/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml +++ b/templates/test/ci/cluster-template-prow-azure-cni-v1.yaml @@ -142,6 +142,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-ci-version.yaml b/templates/test/ci/cluster-template-prow-ci-version.yaml index 94155eefc3a9..bed019bc91b8 100644 --- a/templates/test/ci/cluster-template-prow-ci-version.yaml +++ b/templates/test/ci/cluster-template-prow-ci-version.yaml @@ -217,6 +217,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-custom-vnet.yaml b/templates/test/ci/cluster-template-prow-custom-vnet.yaml index 877f8c4521e3..4ba733ce29ec 100644 --- a/templates/test/ci/cluster-template-prow-custom-vnet.yaml +++ b/templates/test/ci/cluster-template-prow-custom-vnet.yaml @@ -147,6 +147,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-edgezone.yaml b/templates/test/ci/cluster-template-prow-edgezone.yaml index c15c46bab897..43270b2c1c9d 100644 --- a/templates/test/ci/cluster-template-prow-edgezone.yaml +++ b/templates/test/ci/cluster-template-prow-edgezone.yaml @@ -148,6 +148,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml index 6cdd549068c9..8daefed5ef18 100644 --- a/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml +++ b/templates/test/ci/cluster-template-prow-intree-cloud-provider.yaml @@ -156,6 +156,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow-private.yaml b/templates/test/ci/cluster-template-prow-private.yaml index 2aa81dfee959..326a41a6ca57 100644 --- a/templates/test/ci/cluster-template-prow-private.yaml +++ b/templates/test/ci/cluster-template-prow-private.yaml @@ -192,6 +192,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/test/ci/cluster-template-prow.yaml b/templates/test/ci/cluster-template-prow.yaml index e9f64f4c1a9e..d40b731552ad 100644 --- a/templates/test/ci/cluster-template-prow.yaml +++ b/templates/test/ci/cluster-template-prow.yaml @@ -142,6 +142,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/templates/test/dev/cluster-template-custom-builds.yaml b/templates/test/dev/cluster-template-custom-builds.yaml index 95942b75ef89..49923d071e7f 100644 --- a/templates/test/dev/cluster-template-custom-builds.yaml +++ b/templates/test/dev/cluster-template-custom-builds.yaml @@ -208,6 +208,9 @@ spec: apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: + annotations: + cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "5" + cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "1" name: ${CLUSTER_NAME}-md-0 namespace: default spec: diff --git a/test/e2e/azure_csidriver.go b/test/e2e/azure_csidriver.go index 3c1627a581df..04d556efed74 100644 --- a/test/e2e/azure_csidriver.go +++ b/test/e2e/azure_csidriver.go @@ -94,7 +94,7 @@ func AzureDiskCSISpec(ctx context.Context, inputGetter func() AzureDiskCSISpecIn By("creating a deployment that uses pvc") deploymentName := "stateful" + util.RandomString(6) - statefulDeployment := deploymentBuilder.Create("nginx", deploymentName, corev1.NamespaceDefault).AddPVC(pvcName) + statefulDeployment := deploymentBuilder.Create("nginx", deploymentName, corev1.NamespaceDefault, int32(1)).AddPVC(pvcName) deployment, err := statefulDeployment.Deploy(ctx, clientset) Expect(err).NotTo(HaveOccurred()) waitForDeploymentAvailable(ctx, deployment, clientset, specName) diff --git a/test/e2e/azure_lb.go b/test/e2e/azure_lb.go index c98a8bf040df..b21c5116f165 100644 --- a/test/e2e/azure_lb.go +++ b/test/e2e/azure_lb.go @@ -77,7 +77,7 @@ func AzureLBSpec(ctx context.Context, inputGetter func() AzureLBSpecInput) { deploymentName = "web-windows" + util.RandomString(6) } - webDeployment := deploymentBuilder.Create("httpd", deploymentName, corev1.NamespaceDefault) + webDeployment := deploymentBuilder.Create("httpd", deploymentName, corev1.NamespaceDefault, int32(1)) webDeployment.AddContainerPort("http", "http", 80, corev1.ProtocolTCP) if input.Windows { diff --git a/test/e2e/azure_machinepool_drain.go b/test/e2e/azure_machinepool_drain.go index 8c301cc09b4c..fc64241cae98 100644 --- a/test/e2e/azure_machinepool_drain.go +++ b/test/e2e/azure_machinepool_drain.go @@ -274,7 +274,7 @@ func deployHTTPService(ctx context.Context, clientset *kubernetes.Clientset, isW return "web" + util.RandomString(6) }() - webDeploymentBuilder = deployments.Create("httpd", deploymentName, corev1.NamespaceDefault) + webDeploymentBuilder = deployments.Create("httpd", deploymentName, corev1.NamespaceDefault, int32(1)) servicesClient = clientset.CoreV1().Services(corev1.NamespaceDefault) ports = []corev1.ServicePort{ { diff --git a/test/e2e/azure_net_pol.go b/test/e2e/azure_net_pol.go index 18062510861c..3c3f30bc05e3 100644 --- a/test/e2e/azure_net_pol.go +++ b/test/e2e/azure_net_pol.go @@ -96,7 +96,7 @@ func AzureNetPolSpec(ctx context.Context, inputGetter func() AzureNetPolSpecInpu // Front end Prod frontendProdDeploymentName := fmt.Sprintf("frontend-prod-%v", randInt) Log("starting to create frontend-prod deployments") - frontEndProd := deploymentBuilder.Create("library/nginx:1.21.1", frontendProdDeploymentName, namespaceProd.GetName()) + frontEndProd := deploymentBuilder.Create("library/nginx:1.21.1", frontendProdDeploymentName, namespaceProd.GetName(), int32(1)) frontEndProd.AddLabels(frontendLabels) frontendProdDeployment, err := frontEndProd.Deploy(ctx, clientset) Expect(err).NotTo(HaveOccurred()) @@ -104,7 +104,7 @@ func AzureNetPolSpec(ctx context.Context, inputGetter func() AzureNetPolSpecInpu // Front end Dev frontendDevDeploymentName := fmt.Sprintf("frontend-dev-%v", randInt+100000) Log("starting to create frontend-dev deployments") - frontEndDev := deploymentBuilder.Create("library/nginx:1.21.1", frontendDevDeploymentName, namespaceDev.GetName()) + frontEndDev := deploymentBuilder.Create("library/nginx:1.21.1", frontendDevDeploymentName, namespaceDev.GetName(), int32(1)) frontEndDev.AddLabels(frontendLabels) frontendDevDeployment, err := frontEndDev.Deploy(ctx, clientset) Expect(err).NotTo(HaveOccurred()) @@ -113,7 +113,7 @@ func AzureNetPolSpec(ctx context.Context, inputGetter func() AzureNetPolSpecInpu backendDeploymentName := fmt.Sprintf("backend-%v", randInt+200000) backendLabels := map[string]string{"app": "webapp", "role": "backend"} Log("starting to create backend deployments") - backendDev := deploymentBuilder.Create("library/nginx:1.21.1", backendDeploymentName, namespaceDev.GetName()) + backendDev := deploymentBuilder.Create("library/nginx:1.21.1", backendDeploymentName, namespaceDev.GetName(), int32(1)) backendDev.AddLabels(backendLabels) backendDeployment, err := backendDev.Deploy(ctx, clientset) Expect(err).NotTo(HaveOccurred()) @@ -122,7 +122,7 @@ func AzureNetPolSpec(ctx context.Context, inputGetter func() AzureNetPolSpecInpu nwpolicyDeploymentName := fmt.Sprintf("network-policy-%v", randInt+300000) nwpolicyLabels := map[string]string{"app": "webapp", "role": "any"} Log("starting to create network-policy deployments") - nwpolicy := deploymentBuilder.Create("library/nginx:1.21.1", nwpolicyDeploymentName, namespaceDev.GetName()) + nwpolicy := deploymentBuilder.Create("library/nginx:1.21.1", nwpolicyDeploymentName, namespaceDev.GetName(), int32(1)) nwpolicy.AddLabels(nwpolicyLabels) nwpolicyDeployment, err := nwpolicy.Deploy(ctx, clientset) Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/azure_test.go b/test/e2e/azure_test.go index 40f93a4a78d8..444802a124b2 100644 --- a/test/e2e/azure_test.go +++ b/test/e2e/azure_test.go @@ -182,6 +182,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -232,9 +239,26 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) + By("Scaling out via cluster-autoscaler", func() { + AzureAutoscalerSpec(ctx, func() AzureAutoscalerSpecInput { + return AzureAutoscalerSpecInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) + }) + By("Verifying expected VM extensions are present on the node", func() { AzureVMExtensionsSpec(ctx, func() AzureVMExtensionsSpecInput { return AzureVMExtensionsSpecInput{ @@ -294,7 +318,7 @@ var _ = Describe("Workload cluster creation", func() { }) }) - When("Creating a highly available cluster with Azure CNI v1 [REQUIRED]", Label("Azure CNI v1"), func() { + When("Creating a cluster with Azure CNI v1 [REQUIRED]", Label("Azure CNI v1"), func() { It("can create 3 control-plane nodes and 2 Linux worker nodes", func() { clusterName = getClusterName(clusterNamePrefix, "azcni-v1") @@ -377,6 +401,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -415,6 +446,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -475,6 +513,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -561,6 +606,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -669,6 +721,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -847,6 +906,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -921,6 +987,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) @@ -965,6 +1038,13 @@ var _ = Describe("Workload cluster creation", func() { ClusterName: clusterName, } }) + InstallClusterAutoscaler(ctx, func() ClusterAutoscalerInstallInput { + return ClusterAutoscalerInstallInput{ + BootstrapClusterProxy: bootstrapClusterProxy, + Namespace: namespace, + ClusterName: clusterName, + } + }) }), ), result) diff --git a/test/e2e/cluster_autoscaler.go b/test/e2e/cluster_autoscaler.go new file mode 100644 index 000000000000..ce93ddcafec6 --- /dev/null +++ b/test/e2e/cluster_autoscaler.go @@ -0,0 +1,211 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "strconv" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + helmVals "helm.sh/helm/v3/pkg/cli/values" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + deploymentBuilder "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/deployment" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + clusterAutoscalerHelmRepoURL = "https://kubernetes.github.io/autoscaler" + clusterAutoscalerChartName = "cluster-autoscaler" + clusterAutoscalerDeploymentLabelKey = "app.kubernetes.io/name" + clusterAutoscalerDeploymentLabelVal = "clusterapi-cluster-autoscaler" + defaultKubeletMaxPods int32 = 110 +) + +// ClusterAutoscalerInstallInput is the input for InstallClusterAutoscaler. +type ClusterAutoscalerInstallInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + ClusterName string +} + +// InstallClusterAutoscaler implements a test that verifies cluster-autoscaler behaviors. +func InstallClusterAutoscaler(ctx context.Context, inputGetter func() ClusterAutoscalerInstallInput) { + var ( + specName = "clusterAutoscaler" + input ClusterAutoscalerInstallInput + ) + + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + + input = inputGetter() + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) + Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName) + + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + mdList := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ + Lister: mgmtClient, + ClusterName: input.ClusterName, + Namespace: input.Namespace.Name, + }) + var hasClusterAutoscalerConfig bool + for _, md := range mdList { + if _, ok := md.GetAnnotations()["cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size"]; ok { + if _, ok := md.GetAnnotations()["cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"]; ok { + hasClusterAutoscalerConfig = true + } + } + } + if hasClusterAutoscalerConfig { + options := &helmVals.Options{ + Values: []string{ + "cloudProvider=clusterapi", + fmt.Sprintf("autoDiscovery.clusterName=%s", input.ClusterName), + fmt.Sprintf("clusterAPIKubeconfigSecret=%s-kubeconfig", input.ClusterName), + "clusterAPIMode=kubeconfig-incluster", + "extraArgs.scan-interval=1m", + "extraArgs.balance-similar-node-groups=true", + }, + } + InstallHelmChart(ctx, bootstrapClusterProxy, input.Namespace.Name, clusterAutoscalerHelmRepoURL, clusterAutoscalerChartName, fmt.Sprintf("%s-%s", clusterAutoscalerChartName, input.ClusterName), options, "") + } + var d = &appsv1.DeploymentList{} + Eventually(func(g Gomega) bool { + Logf("Listing deployments in namespace %s with label %s=%s", input.Namespace.Name, clusterAutoscalerDeploymentLabelKey, clusterAutoscalerDeploymentLabelVal) + err := mgmtClient.List(ctx, d, client.InNamespace(input.Namespace.Name), client.MatchingLabels{clusterAutoscalerDeploymentLabelKey: clusterAutoscalerDeploymentLabelVal}) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(d).NotTo(BeNil()) + g.Expect(len(d.Items)).To(Equal(1)) + for _, c := range d.Items[0].Status.Conditions { + if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue { + return true + } + } + return false + }, e2eConfig.GetIntervals(specName, "wait-deployment")...).Should(BeTrue()) +} + +// AzureAutoscalerSpecInput is the input for AzureAutoscalerSpec. +type AzureAutoscalerSpecInput struct { + BootstrapClusterProxy framework.ClusterProxy + Namespace *corev1.Namespace + ClusterName string +} + +// AzureVMExtensionsSpec implements a test that verifies VM extensions are created and deleted. +func AzureAutoscalerSpec(ctx context.Context, inputGetter func() AzureAutoscalerSpecInput) { + var ( + specName = "azure-autoscaler" + input AzureAutoscalerSpecInput + ) + + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + + input = inputGetter() + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(input.Namespace).ToNot(BeNil(), "Invalid argument. input.Namespace can't be nil when calling %s spec", specName) + Expect(input.ClusterName).ToNot(BeEmpty(), "Invalid argument. input.ClusterName can't be empty when calling %s spec", specName) + + By("Creating a Kubernetes client to the workload cluster") + workloadClusterProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, input.Namespace.Name, input.ClusterName) + Expect(workloadClusterProxy).NotTo(BeNil()) + mgmtClient := bootstrapClusterProxy.GetClient() + Expect(mgmtClient).NotTo(BeNil()) + clientset := workloadClusterProxy.GetClientSet() + Expect(clientset).NotTo(BeNil()) + + By("Calculating how many pod replicas we need to scale out to maximum") + mdList := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ + Lister: mgmtClient, + ClusterName: input.ClusterName, + Namespace: input.Namespace.Name, + }) + + var clusterAutoscalerMaxNodes int32 + var hasNodeHeadroom bool + for _, md := range mdList { + if val, ok := md.GetAnnotations()["cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"]; ok { + maxSize, err := strconv.ParseInt(val, 10, 64) + Expect(err).NotTo(HaveOccurred()) + if int32(maxSize) > *md.Spec.Replicas { + hasNodeHeadroom = true + } + clusterAutoscalerMaxNodes += int32(maxSize) + } else { + clusterAutoscalerMaxNodes += *md.Spec.Replicas + } + } + if !hasNodeHeadroom { + By("Bypassing cluster-autoscaler scale out because our current node count is not less than our max-size configuration for any pools") + return + } + numReplicas := clusterAutoscalerMaxNodes * defaultKubeletMaxPods + + By(fmt.Sprintf("creating an HTTP deployment with %d replicas", numReplicas)) + deploymentName := "web" + util.RandomString(6) + webDeployment := deploymentBuilder.Create("httpd", deploymentName, corev1.NamespaceDefault, numReplicas) + webDeployment.AddContainerPort("http", "http", 80, corev1.ProtocolTCP) + + deployment, err := webDeployment.Deploy(ctx, clientset) + Expect(err).NotTo(HaveOccurred()) + + By("Verifying that all cluster-autoscaler-enabled MachineDeployments have scaled out") + Eventually(func(g Gomega) { + mdList := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ + Lister: mgmtClient, + ClusterName: input.ClusterName, + Namespace: input.Namespace.Name, + }) + for _, md := range mdList { + if val, ok := md.GetAnnotations()["cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"]; ok { + maxSize, err := strconv.ParseInt(val, 10, 64) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(*md.Spec.Replicas).To(Equal(int32(maxSize))) + g.Expect(md.Status.Phase).To(Equal(string(clusterv1.MachineDeploymentPhaseRunning))) + } + } + }, e2eConfig.GetIntervals(specName, "wait-autoscale-out")...).Should(Succeed()) + By("Deleting deployment in order to delete pod replicas") + err = clientset.AppsV1().Deployments(deployment.GetNamespace()).Delete(ctx, deployment.GetName(), metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred()) + By("Verifying that all cluster-autoscaler-enabled MachineDeployments have scaled in") + Eventually(func(g Gomega) { + mdList := framework.GetMachineDeploymentsByCluster(ctx, framework.GetMachineDeploymentsByClusterInput{ + Lister: mgmtClient, + ClusterName: input.ClusterName, + Namespace: input.Namespace.Name, + }) + for _, md := range mdList { + if val, ok := md.GetAnnotations()["cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size"]; ok { + maxSize, err := strconv.ParseInt(val, 10, 64) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(*md.Spec.Replicas < int32(maxSize)).To(BeTrue()) + } + } + }, e2eConfig.GetIntervals(specName, "wait-autoscale-out")...) +} diff --git a/test/e2e/config/azure-dev.yaml b/test/e2e/config/azure-dev.yaml index beb954c0790b..cb3c058b2795 100644 --- a/test/e2e/config/azure-dev.yaml +++ b/test/e2e/config/azure-dev.yaml @@ -181,7 +181,7 @@ variables: WINDOWS_CONTAINERD_URL: "${WINDOWS_CONTAINERD_URL:-}" SECURITY_SCAN_FAIL_THRESHOLD: "${SECURITY_SCAN_FAIL_THRESHOLD:-100}" SECURITY_SCAN_CONTAINER: "${SECURITY_SCAN_CONTAINER:-quay.io/armosec/kubescape:v2.0.167}" - AZURE_CNI_V1_MANIFEST_PATH: "${PWD}/templates/addons/azure-cni-v1.yaml" + AZURE_CNI_V1_MANIFEST_PATH: "${PWD}/templates/addons/azure-cni-v1.yaml" intervals: default/wait-controllers: ["3m", "10s"] @@ -202,6 +202,7 @@ intervals: default/wait-job: ["5m", "10s"] default/wait-service: ["15m", "10s"] default/wait-machine-pool-nodes: ["30m", "10s"] + default/wait-autoscale-out: ["30m", "10s"] csi-migration/wait-controlplane-upgrade: ["60m", "10s"] csi-migration/wait-worker-nodes: ["60m", "10s"] csi-migration/wait-control-plane: ["60m", "10s"] diff --git a/test/e2e/kubernetes/deployment/deployment.go b/test/e2e/kubernetes/deployment/deployment.go index 876328fa7711..d0b489864188 100644 --- a/test/e2e/kubernetes/deployment/deployment.go +++ b/test/e2e/kubernetes/deployment/deployment.go @@ -54,7 +54,7 @@ type ( ) // Create will create a deployment for a given image with a name in a namespace -func Create(image, name, namespace string) *Builder { +func Create(image, name, namespace string, replicas int32) *Builder { e2eDeployment := &Builder{ deployment: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -63,7 +63,7 @@ func Create(image, name, namespace string) *Builder { Labels: map[string]string{}, }, Spec: appsv1.DeploymentSpec{ - Replicas: pointer.Int32(1), + Replicas: pointer.Int32(replicas), Selector: &metav1.LabelSelector{ MatchLabels: map[string]string{ "app": name, @@ -160,6 +160,19 @@ func (d *Builder) Deploy(ctx context.Context, clientset *kubernetes.Clientset) ( return deployment, nil } +func (d *Builder) Delete(ctx context.Context, clientset *kubernetes.Clientset) error { + Eventually(func(g Gomega) { + var err error + err = d.Client(clientset).Delete(ctx, d.deployment.Name, metav1.DeleteOptions{}) + if err != nil { + log.Printf("Error trying to delete deployment %s in namespace %s:%s\n", d.deployment.Name, d.deployment.ObjectMeta.Namespace, err.Error()) + } + g.Expect(err).NotTo(HaveOccurred()) + }, deploymentOperationTimeout, deploymentOperationSleepBetweenRetries).Should(Succeed()) + + return nil +} + func (d *Builder) Client(clientset *kubernetes.Clientset) typedappsv1.DeploymentInterface { return clientset.AppsV1().Deployments(d.deployment.ObjectMeta.Namespace) }