From f7f653b960d8c105e85449adbddfa18360953847 Mon Sep 17 00:00:00 2001 From: Bob Fournier Date: Wed, 29 May 2024 10:19:48 -0400 Subject: [PATCH] WIP: Add e2e test for internal load balancer Adds an e2e test when the internal load balancer is configured. This requires https://github.com/kubernetes-sigs/cluster-api-provider-gcp/pull/1222 --- controllers/gcpcluster_controller.go | 2 +- test/e2e/config/gcp-ci.yaml | 1 + .../cluster-template-ci-with-internal-lb.yaml | 146 ++++++++++++++++++ test/e2e/e2e_test.go | 24 +++ 4 files changed, 172 insertions(+), 1 deletion(-) create mode 100644 test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml diff --git a/controllers/gcpcluster_controller.go b/controllers/gcpcluster_controller.go index f5eddfe1b..10f077813 100644 --- a/controllers/gcpcluster_controller.go +++ b/controllers/gcpcluster_controller.go @@ -231,8 +231,8 @@ func (r *GCPClusterReconciler) reconcileDelete(ctx context.Context, clusterScope log.Info("Reconciling Delete GCPCluster") reconcilers := []cloud.Reconciler{ - subnets.New(clusterScope), loadbalancers.New(clusterScope), + subnets.New(clusterScope), firewalls.New(clusterScope), networks.New(clusterScope), } diff --git a/test/e2e/config/gcp-ci.yaml b/test/e2e/config/gcp-ci.yaml index 058bff220..39abbe42c 100644 --- a/test/e2e/config/gcp-ci.yaml +++ b/test/e2e/config/gcp-ci.yaml @@ -72,6 +72,7 @@ providers: - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke.yaml" - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-autopilot.yaml" - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-custom-subnet.yaml" + - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml" variables: KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.29.0}" diff --git a/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml new file mode 100644 index 000000000..7ce0e2427 --- /dev/null +++ b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-internal-lb.yaml @@ -0,0 +1,146 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + cni: "${CLUSTER_NAME}-crs-cni" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + project: "${GCP_PROJECT}" + region: "${GCP_REGION}" + network: + name: "${GCP_NETWORK_NAME}" + subnets: + - name: control-plane-subnet + cidrBlock: "10.0.0.0/17" + purpose: PRIVATE + region: us-east4 + loadBalancer: + loadBalancerType: InternalExternal +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: GCPMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + useExperimentalRetryJoin: true + initConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce + clusterConfiguration: + apiServer: + timeoutForControlPlane: 20m + extraArgs: + cloud-provider: gce + controllerManager: + extraArgs: + cloud-provider: gce + allocate-node-cidrs: "false" + kubernetesVersion: "${KUBERNETES_VERSION}" + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + instanceType: "${GCP_NODE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "${CLUSTER_NAME}-crs-cni" +data: ${CNI_RESOURCES} +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: "${CLUSTER_NAME}-crs-cni" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + cni: "${CLUSTER_NAME}-crs-cni" + resources: + - name: "${CLUSTER_NAME}-crs-cni" + kind: ConfigMap diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index cf36a585a..906ae0698 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -182,4 +182,28 @@ var _ = Describe("Workload cluster creation", func() { }, result) }) }) + + Context("Creating a control-plane cluster with an internal load balancer", func() { + It("Should create a cluster with 1 control-plane and 1 worker node with an internal load balancer", func() { + By("Creating a cluster with internal load balancer") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "ci-with-internal-lb", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) })