Add Clusterctl Move E2E test

Also set ownerReference on trust bundle configmaps so CMs get moved
when clusterctl move happens.

.gitignore

@@ -7,6 +7,7 @@
 *.dylib
 bin
 testbin/*
+*.tmp
 
 # Test binary, build with `go test -c`
 *.test

Makefile

@@ -4,18 +4,17 @@ GOTEST=$(GOCMD) test
 GOGET=$(GOCMD) get
 GOTOOL=$(GOCMD) tool
 EXPORT_RESULT?=false # for CI please set EXPORT_RESULT to true
-# Image URL to use all building/pushing image targets
-IMG ?= ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest
+
+GIT_COMMIT_HASH=$(shell git rev-parse HEAD)
+LOCAL_IMAGE_REGISTRY ?= localhost:5000
+IMG_REPO=${LOCAL_IMAGE_REGISTRY}/cluster-api-provider-nutanix
+IMG_TAG=e2e-${GIT_COMMIT_HASH}
+MANAGER_IMAGE=${IMG_REPO}:${IMG_TAG}
 
 # Extract base and tag from IMG
-IMG_REPO ?= $(word 1,$(subst :, ,${IMG}))
-IMG_TAG ?= $(word 2,$(subst :, ,${IMG}))
 LOCAL_PROVIDER_VERSION ?= ${IMG_TAG}
-ifeq (${IMG_TAG},)
-IMG_TAG := latest
-endif
 
-ifeq (${LOCAL_PROVIDER_VERSION},latest)
+ifeq (${LOCAL_PROVIDER_VERSION},${IMG_TAG})
 # TODO(release-blocker): Change this versions after release when required here and in e2e config (test/e2e/config/nutanix.yaml)
 LOCAL_PROVIDER_VERSION := v1.4.99
 endif
@@ -62,21 +61,6 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-# Get latest git hash
-GIT_COMMIT_HASH=$(shell git rev-parse HEAD)
-
-# Get the local image registry required for clusterctl upgrade tests
-LOCAL_IMAGE_REGISTRY ?= localhost:5000
-
-ifeq (${MAKECMDGOALS},test-e2e-clusterctl-upgrade)
-	IMG_TAG=e2e-${GIT_COMMIT_HASH}
-	IMG_REPO=${LOCAL_IMAGE_REGISTRY}/controller
-endif
-
-ifeq (${MAKECMDGOALS},docker-build-e2e)
-	IMG_TAG=e2e-${GIT_COMMIT_HASH}
-endif
-
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # This is a requirement for 'setup-envtest.sh' in the test target.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
@@ -94,10 +78,11 @@ ifneq ($(LABEL_FILTERS),)
 		LABEL_FILTER_ARGS := "$(LABEL_FILTER_ARGS) && $(LABEL_FILTERS)"
 endif
 JUNIT_REPORT_FILE ?= "junit.e2e_suite.1.xml"
-GINKGO_SKIP ?= "clusterctl-Upgrade"
+GINKGO_SKIP ?= ""
 GINKGO_FOCUS ?= ""
 GINKGO_NODES  ?= 1
 E2E_CONF_FILE  ?= ${E2E_DIR}/config/nutanix.yaml
+E2E_CONF_FILE_TMP = ${E2E_CONF_FILE}.tmp
 ARTIFACTS ?= ${REPO_ROOT}/_artifacts
 SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false
@@ -287,7 +272,7 @@ cluster-templates: ## Generate cluster templates for all flavors
 docker-build-e2e: ## Build docker image with the manager with e2e tag.
 	echo "Git commit hash: ${GIT_COMMIT_HASH}"
 	KO_DOCKER_REPO=ko.local GOFLAGS="-ldflags=-X=main.gitCommitHash=${GIT_COMMIT_HASH}" ko build -B --platform=${PLATFORMS_E2E} -t ${IMG_TAG} .
-	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:e2e
+	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
 
 .PHONY: prepare-local-clusterctl
 prepare-local-clusterctl: manifests cluster-templates  ## Prepare overide file for local clusterctl.
@@ -332,6 +317,10 @@ template-test: cluster-templates ## Run the template tests
 
 .PHONY: test-e2e
 test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
+	echo "Image tag for E2E test is ${IMG_TAG}"
+	MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
+	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
+	docker push ${IMG_REPO}:${IMG_TAG}
 	mkdir -p $(ARTIFACTS)
 	NUTANIX_LOG_LEVEL=debug ginkgo -v \
 		--trace \
@@ -348,12 +337,16 @@ test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en
 		--always-emit-ginkgo-writer \
 		$(GINKGO_ARGS) ./test/e2e -- \
 		-e2e.artifacts-folder="$(ARTIFACTS)" \
-		-e2e.config="$(E2E_CONF_FILE)" \
+		-e2e.config="$(E2E_CONF_FILE_TMP)" \
 		-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
 
 .PHONY: test-e2e-no-kubeproxy
 test-e2e-no-kubeproxy: docker-build-e2e cluster-e2e-templates-no-kubeproxy cluster-templates ## Run the end-to-end tests without kubeproxy
+	echo "Image tag for E2E test is ${IMG_TAG}"
+	MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
+	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
+	docker push ${IMG_REPO}:${IMG_TAG}
 	mkdir -p $(ARTIFACTS)
 	NUTANIX_LOG_LEVEL=debug ginkgo -v \
 		--trace \
@@ -385,15 +378,15 @@ list-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en
 
 .PHONY: test-e2e-calico
 test-e2e-calico:
-	CNI=$(CNI_PATH_CALICO) $(MAKE) test-e2e
+	CNI=$(CNI_PATH_CALICO) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e
 
 .PHONY: test-e2e-flannel
 test-e2e-flannel:
-	CNI=$(CNI_PATH_FLANNEL) $(MAKE) test-e2e
+	CNI=$(CNI_PATH_FLANNEL) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e
 
 .PHONY: test-e2e-cilium
 test-e2e-cilium:
-	CNI=$(CNI_PATH_CILIUM) $(MAKE) test-e2e
+	CNI=$(CNI_PATH_CILIUM) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e
 
 .PHONY: test-e2e-cilium-no-kubeproxy
 test-e2e-cilium-no-kubeproxy:
@@ -402,13 +395,6 @@ test-e2e-cilium-no-kubeproxy:
 .PHONY: test-e2e-all-cni
 test-e2e-all-cni: test-e2e test-e2e-calico test-e2e-flannel test-e2e-cilium test-e2e-cilium-no-kubeproxy
 
-.PHONY: test-e2e-clusterctl-upgrade
-test-e2e-clusterctl-upgrade: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
-	echo "Image tag for E2E test is ${IMG_TAG}"
-	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
-	docker push ${IMG_REPO}:${IMG_TAG}
-	GINKGO_SKIP="" GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e-calico
-
 ##@ Lint and Verify
 
 .PHONY: lint

api/v1beta1/conditions.go

@@ -74,5 +74,7 @@ const (
 	// CredentialRefSecretOwnerSetCondition shows the status of setting the Owner
 	CredentialRefSecretOwnerSetCondition capiv1.ConditionType = "CredentialRefSecretOwnerSet"
 
-	CredentialRefSecretOwnerSetFailed = "CredentialRefSecretOwnerSetFailed"
+	CredentialRefSecretOwnerSetFailed  = "CredentialRefSecretOwnerSetFailed"
+	TrustBundleSecretOwnerSetCondition = "TrustBundleSecretOwnerSet"
+	TrustBundleSecretOwnerSetFailed    = "TrustBundleSecretOwnerSetFailed"
 )

api/v1beta1/nutanixcluster_types.go

@@ -164,6 +164,17 @@ func (ncl *NutanixCluster) GetPrismCentralCredentialRef() (*credentialTypes.Nuta
 	return prismCentralInfo.CredentialRef, nil
 }
 
+func (ncl *NutanixCluster) GetPrismCentralTrustBundle() *credentialTypes.NutanixTrustBundleReference {
+	prismCentralInfo := ncl.Spec.PrismCentral
+	if prismCentralInfo == nil ||
+		prismCentralInfo.AdditionalTrustBundle == nil ||
+		prismCentralInfo.AdditionalTrustBundle.Kind == credentialTypes.NutanixTrustBundleKindString {
+		return nil
+	}
+
+	return prismCentralInfo.AdditionalTrustBundle
+}
+
 // GetNamespacedName returns the namespaced name of the NutanixCluster.
 func (ncl *NutanixCluster) GetNamespacedName() string {
 	namespace := cmp.Or(ncl.Namespace, corev1.NamespaceDefault)

controllers/nutanixcluster_controller.go

@@ -17,6 +17,7 @@ limitations under the License.
 package controllers
 
 import (
+	"cmp"
 	"context"
 	"fmt"
 	"time"
@@ -178,6 +179,13 @@ func (r *NutanixClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 	conditions.MarkTrue(cluster, infrav1.CredentialRefSecretOwnerSetCondition)
 
+	if err := r.reconcileTrustBundleRef(ctx, cluster); err != nil {
+		log.Error(err, fmt.Sprintf("error occurred while reconciling trust bundle ref for cluster %s", capiCluster.Name))
+		conditions.MarkFalse(cluster, infrav1.TrustBundleSecretOwnerSetCondition, infrav1.TrustBundleSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
+		return reconcile.Result{}, err
+	}
+	conditions.MarkTrue(cluster, infrav1.TrustBundleSecretOwnerSetCondition)
+
 	v3Client, err := getPrismCentralClientForCluster(ctx, cluster, r.SecretInformer, r.ConfigMapInformer)
 	if err != nil {
 		log.Error(err, "error occurred while fetching prism central client")
@@ -373,6 +381,42 @@ func (r *NutanixClusterReconciler) reconcileCredentialRefDelete(ctx context.Cont
 	return nil
 }
 
+func (r *NutanixClusterReconciler) reconcileTrustBundleRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
+	log := ctrl.LoggerFrom(ctx)
+	trustBundleRef := nutanixCluster.GetPrismCentralTrustBundle()
+	if trustBundleRef == nil {
+		log.Info(fmt.Sprintf("trust bundle ref is nil for cluster %s", nutanixCluster.Name))
+		return nil
+	}
+
+	// get the trust bundle configmap
+	configMap := &corev1.ConfigMap{}
+	configMapKey := client.ObjectKey{
+		Namespace: cmp.Or(trustBundleRef.Namespace, nutanixCluster.Namespace),
+		Name:      trustBundleRef.Name,
+	}
+	if err := r.Client.Get(ctx, configMapKey, configMap); err != nil {
+		log.Error(err, "error occurred while fetching trust bundle configmap", "nutanixCluster", nutanixCluster.Name)
+		return err
+	}
+
+	if !capiutil.IsOwnedByObject(configMap, nutanixCluster) {
+		configMap.OwnerReferences = capiutil.EnsureOwnerRef(configMap.OwnerReferences, metav1.OwnerReference{
+			APIVersion: infrav1.GroupVersion.String(),
+			Kind:       nutanixCluster.Kind,
+			UID:        nutanixCluster.UID,
+			Name:       nutanixCluster.Name,
+		})
+	}
+
+	if err := r.Client.Update(ctx, configMap); err != nil {
+		log.Error(err, "error occurred while updating trust bundle configmap", "nutanixCluster", nutanixCluster)
+		return err
+	}
+
+	return nil
+}
+
 func (r *NutanixClusterReconciler) reconcileCredentialRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
 	log := ctrl.LoggerFrom(ctx)
 	credentialRef, err := getPrismCentralCredentialRefForCluster(nutanixCluster)

test/e2e/clusterctl_move_test.go

@@ -0,0 +1,55 @@
+//go:build e2e
+
+/*
+Copyright 2024 Nutanix
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	capie2e "sigs.k8s.io/cluster-api/test/e2e"
+)
+
+var _ = Describe("[clusterctl-move] Create a Self-Hosted CAPX Cluster", Label("capx-feature-test", "clusterctl-move"), func() {
+	// Run the CAPI SelfHostedSpec test suite
+	// This test suite will create a self-hosted CAPX cluster
+	capie2e.SelfHostedSpec(ctx, func() capie2e.SelfHostedSpecInput {
+		return capie2e.SelfHostedSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+			SkipUpgrade:           true,
+		}
+	})
+})
+
+var _ = Describe("[clusterctl-move] Create a Self-Hosted CAPX Cluster", Label("clusterclass", "clusterctl-move"), func() {
+	// Run the CAPI SelfHostedSpec test suite
+	// This test suite will create a self-hosted CAPX cluster
+	capie2e.SelfHostedSpec(ctx, func() capie2e.SelfHostedSpecInput {
+		return capie2e.SelfHostedSpecInput{
+			E2EConfig:             e2eConfig,
+			Flavor:                flavorTopology,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+			SkipUpgrade:           true,
+		}
+	})
+})

test/e2e/config/nutanix.yaml

@@ -9,7 +9,7 @@
 
 images:
   # Use local dev images built source tree;
-  - name: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:e2e
+  - name: ${MANAGER_IMAGE}
     loadBehavior: mustLoad
   # ## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS
   # Cluster API v1beta1 Preloads
@@ -193,10 +193,10 @@ providers:
         value: "../../../config/default"
         contract: v1beta1
         replacements:
-          - old: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest
-            new: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:e2e
           - old: "imagePullPolicy: Always"
             new: "imagePullPolicy: IfNotPresent"
+          - old: "image: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest"
+            new: "image: ${MANAGER_IMAGE}"
         files:
           - sourcePath: "../../../metadata.yaml"
           - sourcePath: "../data/infrastructure-nutanix/v1beta1/cluster-template.yaml"

test/e2e/test_helpers.go

@@ -70,6 +70,8 @@ const (
 	nameType = "name"
 
 	nutanixProjectNameEnv = "NUTANIX_PROJECT_NAME"
+
+	flavorTopology = "topology"
 )
 
 // Test suite global vars.