From 9c44f1a757309284e96d80c0694cf044b06a9a6d Mon Sep 17 00:00:00 2001 From: Brent Barbachem Date: Mon, 3 Jun 2024 10:50:21 -0400 Subject: [PATCH] Adding e2e tests for shared vpc installs --- scripts/ci-e2e.sh | 96 ++++++++----- test/e2e/config/gcp-ci.yaml | 1 + .../cluster-template-ci-with-shared-vpc.yaml | 127 ++++++++++++++++++ test/e2e/e2e_test.go | 25 ++++ 4 files changed, 214 insertions(+), 35 deletions(-) create mode 100644 test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml diff --git a/scripts/ci-e2e.sh b/scripts/ci-e2e.sh index ce298a265..5c0d69b4f 100755 --- a/scripts/ci-e2e.sh +++ b/scripts/ci-e2e.sh @@ -102,27 +102,38 @@ EOF # initialize a router and cloud NAT init_networks() { - if [[ ${GCP_NETWORK_NAME} != "default" ]]; then - gcloud compute networks create --project "$GCP_PROJECT" "${GCP_NETWORK_NAME}" --subnet-mode auto --quiet - gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-http --project "$GCP_PROJECT" \ - --allow tcp:80 --network "${GCP_NETWORK_NAME}" --quiet - gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-https --project "$GCP_PROJECT" \ - --allow tcp:443 --network "${GCP_NETWORK_NAME}" --quiet - gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-icmp --project "$GCP_PROJECT" \ - --allow icmp --network "${GCP_NETWORK_NAME}" --priority 65534 --quiet - gcloud compute firewall-rules create "${GCP_NETWORK_NAME}"-allow-internal --project "$GCP_PROJECT" \ - --allow "tcp:0-65535,udp:0-65535,icmp" --network "${GCP_NETWORK_NAME}" --priority 65534 --quiet - fi - gcloud compute firewall-rules list --project "$GCP_PROJECT" - gcloud compute networks list --project="${GCP_PROJECT}" - gcloud compute networks describe "${GCP_NETWORK_NAME}" --project="${GCP_PROJECT}" + ARRAY=( "$GCP_PROJECT:$GCP_NETWORK_NAME" + "$GCP_HOST_PROJECT:${GCP_NETWORK_NAME}-shared-vpc" ) + + for data in "${ARRAY[@]}"; do + PROJECT=${data%%:*} + NETWORK=${data#*:} + + echo "Creating network resources in project ${PROJECT}" + + if [[ ${NETWORK} != "default" ]]; then + gcloud compute networks create --project "$PROJECT" "${NETWORK}" --subnet-mode auto --quiet + gcloud compute firewall-rules create "${NETWORK}"-allow-http --project "$PROJECT" \ + --allow tcp:80 --network "${NETWORK}" --quiet + gcloud compute firewall-rules create "${NETWORK}"-allow-https --project "$PROJECT" \ + --allow tcp:443 --network "${NETWORK}" --quiet + gcloud compute firewall-rules create "${NETWORK}"-allow-icmp --project "$PROJECT" \ + --allow icmp --network "${NETWORK}" --priority 65534 --quiet + gcloud compute firewall-rules create "${NETWORK}"-allow-internal --project "$PROJECT" \ + --allow "tcp:0-65535,udp:0-65535,icmp" --network "${NETWORK}" --priority 65534 --quiet + fi + + gcloud compute firewall-rules list --project "$PROJECT" + gcloud compute networks list --project="${$PROJECT}" + gcloud compute networks describe "${NETWORK}" --project="${$PROJECT}" - gcloud compute routers create "${TEST_NAME}-myrouter" --project="${GCP_PROJECT}" \ - --region="${GCP_REGION}" --network="${GCP_NETWORK_NAME}" - gcloud compute routers nats create "${TEST_NAME}-mynat" --project="${GCP_PROJECT}" \ - --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" \ - --nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips + gcloud compute routers create "${TEST_NAME}-myrouter" --project="${$PROJECT}" \ + --region="${GCP_REGION}" --network="${NETWORK}" + gcloud compute routers nats create "${TEST_NAME}-mynat" --project="${$PROJECT}" \ + --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" \ + --nat-all-subnet-ip-ranges --auto-allocate-nat-external-ips + done } @@ -146,23 +157,38 @@ cleanup() { (gcloud compute instance-groups list --project "$GCP_PROJECT" | grep capg-e2e \ | awk '{print "gcloud compute instance-groups unmanaged delete --project '"$GCP_PROJECT"' --quiet " $1 " --zone " $2 "\n"}' \ | bash) || true - (gcloud compute firewall-rules list --project "$GCP_PROJECT" | grep capg-e2e \ - | awk '{print "gcloud compute firewall-rules delete --project '"$GCP_PROJECT"' --quiet " $1 "\n"}' \ - | bash) || true - # cleanup the networks - gcloud compute routers nats delete "${TEST_NAME}-mynat" --project="${GCP_PROJECT}" \ - --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" --quiet || true - gcloud compute routers delete "${TEST_NAME}-myrouter" --project="${GCP_PROJECT}" \ - --region="${GCP_REGION}" --quiet || true - - if [[ ${GCP_NETWORK_NAME} != "default" ]]; then - (gcloud compute firewall-rules list --project "$GCP_PROJECT" | grep "$GCP_NETWORK_NAME" \ - | awk '{print "gcloud compute firewall-rules delete --project '"$GCP_PROJECT"' --quiet " $1 "\n"}' \ - | bash) || true - gcloud compute networks delete --project="${GCP_PROJECT}" \ - --quiet "${GCP_NETWORK_NAME}" || true - fi + + ARRAY=( "$GCP_PROJECT:$GCP_NETWORK_NAME" + "$GCP_HOST_PROJECT:${GCP_NETWORK_NAME}-shared-vpc" ) + + for data in "${ARRAY[@]}"; do + PROJECT=${data%%:*} + NETWORK=${data#*:} + + echo "Cleaning up network resources from project $PROJECT" + + (gcloud compute firewall-rules list --project "$PROJECT" | grep capg-e2e \ + | awk '{print "gcloud compute firewall-rules delete --project '"$PROJECT"' --quiet " $1 "\n"}' \ + | bash) || true + + # cleanup the networks + gcloud compute routers nats delete "${TEST_NAME}-mynat" --project="${PROJECT}" \ + --router-region="${GCP_REGION}" --router="${TEST_NAME}-myrouter" --quiet || true + gcloud compute routers delete "${TEST_NAME}-myrouter" --project="${PROJECT}" \ + --region="${GCP_REGION}" --quiet || true + + if [[ ${NETWORK} != "default" ]]; then + (gcloud compute firewall-rules list --project "$PROJECT" | grep "$NETWORK" \ + | awk '{print "gcloud compute firewall-rules delete --project '"$PROJECT"' --quiet " $1 "\n"}' \ + | bash) || true + gcloud compute networks delete --project="${PROJECT}" \ + --quiet "${NETWORK}" || true + fi + + done + + if [[ -n "${SKIP_INIT_IMAGE:-}" ]]; then echo "Skipping GCP image deletion..." diff --git a/test/e2e/config/gcp-ci.yaml b/test/e2e/config/gcp-ci.yaml index 058bff220..e32d3ccb5 100644 --- a/test/e2e/config/gcp-ci.yaml +++ b/test/e2e/config/gcp-ci.yaml @@ -72,6 +72,7 @@ providers: - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke.yaml" - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-autopilot.yaml" - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-gke-custom-subnet.yaml" + - sourcePath: "${PWD}/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml" variables: KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.29.0}" diff --git a/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml new file mode 100644 index 000000000..39d04e49a --- /dev/null +++ b/test/e2e/data/infrastructure-gcp/cluster-template-ci-with-shared-vpc.yaml @@ -0,0 +1,127 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + cni: "${CLUSTER_NAME}-shared-vpc" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + project: "${GCP_PROJECT}" + region: "${GCP_REGION}" + network: + name: "${GCP_NETWORK_NAME}" + hostProject: "${GCP_HOST_PROJECT}" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: GCPMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + useExperimentalRetryJoin: true + initConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce + clusterConfiguration: + apiServer: + timeoutForControlPlane: 20m + extraArgs: + cloud-provider: gce + controllerManager: + extraArgs: + cloud-provider: gce + allocate-node-cidrs: "false" + kubernetesVersion: "${KUBERNETES_VERSION}" + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce + version: "${KUBERNETES_VERSION}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + instanceType: "${GCP_CONTROL_PLANE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: GCPMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: GCPMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + instanceType: "${GCP_NODE_MACHINE_TYPE}" + image: "${IMAGE_ID}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + name: '{{ ds.meta_data.local_hostname.split(".")[0] }}' + kubeletExtraArgs: + cloud-provider: gce +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "${CLUSTER_NAME}-shared-vpc" +data: ${CNI_RESOURCES} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index cf36a585a..92b8a2b8d 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -182,4 +182,29 @@ var _ = Describe("Workload cluster creation", func() { }, result) }) }) + + Context("Creating a control-plane cluster with a shared vpc", func() { + It("Should create a cluster with 1 control-plane and 1 worker node where the network exists in a host project", func() { + By("Creating a cluster where the host project shares network resources with the service project") + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: bootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: clusterctlConfigPath, + KubeconfigPath: bootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: "ci-with-shared-vpc", + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: e2eConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + }, + WaitForClusterIntervals: e2eConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: e2eConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: e2eConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + }) + }) + })