Add Clusterctl Move E2E test (#438)

Also set ownerReference on trust bundle configmaps so CMs get moved
when clusterctl move happens.

.gitignore

@@ -7,6 +7,7 @@
 *.dylib
 bin
 testbin/*
+*.tmp
 
 # Test binary, build with `go test -c`
 *.test

Makefile

@@ -4,18 +4,17 @@ GOTEST=$(GOCMD) test
 GOGET=$(GOCMD) get
 GOTOOL=$(GOCMD) tool
 EXPORT_RESULT?=false # for CI please set EXPORT_RESULT to true
-# Image URL to use all building/pushing image targets
-IMG ?= ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest
+
+GIT_COMMIT_HASH=$(shell git rev-parse HEAD)
+LOCAL_IMAGE_REGISTRY ?= localhost:5000
+IMG_REPO=${LOCAL_IMAGE_REGISTRY}/cluster-api-provider-nutanix
+IMG_TAG=e2e-${GIT_COMMIT_HASH}
+MANAGER_IMAGE=${IMG_REPO}:${IMG_TAG}
 
 # Extract base and tag from IMG
-IMG_REPO ?= $(word 1,$(subst :, ,${IMG}))
-IMG_TAG ?= $(word 2,$(subst :, ,${IMG}))
 LOCAL_PROVIDER_VERSION ?= ${IMG_TAG}
-ifeq (${IMG_TAG},)
-IMG_TAG := latest
-endif
 
-ifeq (${LOCAL_PROVIDER_VERSION},latest)
+ifeq (${LOCAL_PROVIDER_VERSION},${IMG_TAG})
 # TODO(release-blocker): Change this versions after release when required here and in e2e config (test/e2e/config/nutanix.yaml)
 LOCAL_PROVIDER_VERSION := v1.4.99
 endif
@@ -62,21 +61,6 @@ else
 GOBIN=$(shell go env GOBIN)
 endif
 
-# Get latest git hash
-GIT_COMMIT_HASH=$(shell git rev-parse HEAD)
-
-# Get the local image registry required for clusterctl upgrade tests
-LOCAL_IMAGE_REGISTRY ?= localhost:5000
-
-ifeq (${MAKECMDGOALS},test-e2e-clusterctl-upgrade)
-	IMG_TAG=e2e-${GIT_COMMIT_HASH}
-	IMG_REPO=${LOCAL_IMAGE_REGISTRY}/controller
-endif
-
-ifeq (${MAKECMDGOALS},docker-build-e2e)
-	IMG_TAG=e2e-${GIT_COMMIT_HASH}
-endif
-
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # This is a requirement for 'setup-envtest.sh' in the test target.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
@@ -94,10 +78,11 @@ ifneq ($(LABEL_FILTERS),)
 		LABEL_FILTER_ARGS := "$(LABEL_FILTER_ARGS) && $(LABEL_FILTERS)"
 endif
 JUNIT_REPORT_FILE ?= "junit.e2e_suite.1.xml"
-GINKGO_SKIP ?= "clusterctl-Upgrade"
+GINKGO_SKIP ?= ""
 GINKGO_FOCUS ?= ""
 GINKGO_NODES  ?= 1
 E2E_CONF_FILE  ?= ${E2E_DIR}/config/nutanix.yaml
+E2E_CONF_FILE_TMP = ${E2E_CONF_FILE}.tmp
 ARTIFACTS ?= ${REPO_ROOT}/_artifacts
 SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false
@@ -287,7 +272,7 @@ cluster-templates: ## Generate cluster templates for all flavors
 docker-build-e2e: ## Build docker image with the manager with e2e tag.
 	echo "Git commit hash: ${GIT_COMMIT_HASH}"
 	KO_DOCKER_REPO=ko.local GOFLAGS="-ldflags=-X=main.gitCommitHash=${GIT_COMMIT_HASH}" ko build -B --platform=${PLATFORMS_E2E} -t ${IMG_TAG} .
-	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:e2e
+	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
 
 .PHONY: prepare-local-clusterctl
 prepare-local-clusterctl: manifests cluster-templates  ## Prepare overide file for local clusterctl.
@@ -332,6 +317,10 @@ template-test: cluster-templates ## Run the template tests
 
 .PHONY: test-e2e
 test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
+	echo "Image tag for E2E test is ${IMG_TAG}"
+	MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
+	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
+	docker push ${IMG_REPO}:${IMG_TAG}
 	mkdir -p $(ARTIFACTS)
 	NUTANIX_LOG_LEVEL=debug ginkgo -v \
 		--trace \
@@ -348,12 +337,16 @@ test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en
 		--always-emit-ginkgo-writer \
 		$(GINKGO_ARGS) ./test/e2e -- \
 		-e2e.artifacts-folder="$(ARTIFACTS)" \
-		-e2e.config="$(E2E_CONF_FILE)" \
+		-e2e.config="$(E2E_CONF_FILE_TMP)" \
 		-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)
 
 .PHONY: test-e2e-no-kubeproxy
 test-e2e-no-kubeproxy: docker-build-e2e cluster-e2e-templates-no-kubeproxy cluster-templates ## Run the end-to-end tests without kubeproxy
+	echo "Image tag for E2E test is ${IMG_TAG}"
+	MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
+	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
+	docker push ${IMG_REPO}:${IMG_TAG}
 	mkdir -p $(ARTIFACTS)
 	NUTANIX_LOG_LEVEL=debug ginkgo -v \
 		--trace \
@@ -385,15 +378,15 @@ list-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en
 
 .PHONY: test-e2e-calico
 test-e2e-calico:
-	CNI=$(CNI_PATH_CALICO) $(MAKE) test-e2e
+	CNI=$(CNI_PATH_CALICO) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e
 
 .PHONY: test-e2e-flannel
 test-e2e-flannel:
-	CNI=$(CNI_PATH_FLANNEL) $(MAKE) test-e2e
+	CNI=$(CNI_PATH_FLANNEL) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e
 
 .PHONY: test-e2e-cilium
 test-e2e-cilium:
-	CNI=$(CNI_PATH_CILIUM) $(MAKE) test-e2e
+	CNI=$(CNI_PATH_CILIUM) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e
 
 .PHONY: test-e2e-cilium-no-kubeproxy
 test-e2e-cilium-no-kubeproxy:
@@ -402,13 +395,6 @@ test-e2e-cilium-no-kubeproxy:
 .PHONY: test-e2e-all-cni
 test-e2e-all-cni: test-e2e test-e2e-calico test-e2e-flannel test-e2e-cilium test-e2e-cilium-no-kubeproxy
 
-.PHONY: test-e2e-clusterctl-upgrade
-test-e2e-clusterctl-upgrade: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
-	echo "Image tag for E2E test is ${IMG_TAG}"
-	docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
-	docker push ${IMG_REPO}:${IMG_TAG}
-	GINKGO_SKIP="" GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e-calico
-
 ##@ Lint and Verify
 
 .PHONY: lint

api/v1beta1/conditions.go

@@ -74,5 +74,7 @@ const (
 	// CredentialRefSecretOwnerSetCondition shows the status of setting the Owner
 	CredentialRefSecretOwnerSetCondition capiv1.ConditionType = "CredentialRefSecretOwnerSet"
 
-	CredentialRefSecretOwnerSetFailed = "CredentialRefSecretOwnerSetFailed"
+	CredentialRefSecretOwnerSetFailed  = "CredentialRefSecretOwnerSetFailed"
+	TrustBundleSecretOwnerSetCondition = "TrustBundleSecretOwnerSet"
+	TrustBundleSecretOwnerSetFailed    = "TrustBundleSecretOwnerSetFailed"
 )

api/v1beta1/nutanixcluster_types.go

@@ -164,6 +164,18 @@ func (ncl *NutanixCluster) GetPrismCentralCredentialRef() (*credentialTypes.Nuta
 	return prismCentralInfo.CredentialRef, nil
 }
 
+// GetPrismCentralTrustBundle returns the trust bundle reference for the Nutanix Prism Central.
+func (ncl *NutanixCluster) GetPrismCentralTrustBundle() *credentialTypes.NutanixTrustBundleReference {
+	prismCentralInfo := ncl.Spec.PrismCentral
+	if prismCentralInfo == nil ||
+		prismCentralInfo.AdditionalTrustBundle == nil ||
+		prismCentralInfo.AdditionalTrustBundle.Kind == credentialTypes.NutanixTrustBundleKindString {
+		return nil
+	}
+
+	return prismCentralInfo.AdditionalTrustBundle
+}
+
 // GetNamespacedName returns the namespaced name of the NutanixCluster.
 func (ncl *NutanixCluster) GetNamespacedName() string {
 	namespace := cmp.Or(ncl.Namespace, corev1.NamespaceDefault)

api/v1beta1/nutanixcluster_types_test.go

@@ -148,3 +148,68 @@ func TestGetNamespacedName(t *testing.T) {
 		})
 	}
 }
+
+func TestGetPrismCentralTrustBundle(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name                string
+		nutanixCluster      *NutanixCluster
+		expectedTrustBundle *credentials.NutanixTrustBundleReference
+	}{
+		{
+			name: "PrismCentral and AdditionalTrustBundle are nil",
+			nutanixCluster: &NutanixCluster{
+				Spec: NutanixClusterSpec{},
+			},
+			expectedTrustBundle: nil,
+		},
+		{
+			name: "AdditionalTrustBundle is nil",
+			nutanixCluster: &NutanixCluster{
+				Spec: NutanixClusterSpec{
+					PrismCentral: &credentials.NutanixPrismEndpoint{},
+				},
+			},
+			expectedTrustBundle: nil,
+		},
+		{
+			name: "AdditionalTrustBundle Kind is NutanixTrustBundleKindString",
+			nutanixCluster: &NutanixCluster{
+				Spec: NutanixClusterSpec{
+					PrismCentral: &credentials.NutanixPrismEndpoint{
+						AdditionalTrustBundle: &credentials.NutanixTrustBundleReference{
+							Kind: credentials.NutanixTrustBundleKindString,
+						},
+					},
+				},
+			},
+			expectedTrustBundle: nil,
+		},
+		{
+			name: "AdditionalTrustBundle is not nil and Kind is not NutanixTrustBundleKindString",
+			nutanixCluster: &NutanixCluster{
+				Spec: NutanixClusterSpec{
+					PrismCentral: &credentials.NutanixPrismEndpoint{
+						AdditionalTrustBundle: &credentials.NutanixTrustBundleReference{
+							Kind:      credentials.NutanixTrustBundleKindConfigMap,
+							Name:      "test",
+							Namespace: "test-namespace",
+						},
+					},
+				},
+			},
+			expectedTrustBundle: &credentials.NutanixTrustBundleReference{
+				Kind:      credentials.NutanixTrustBundleKindConfigMap,
+				Name:      "test",
+				Namespace: "test-namespace",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			trustBundle := tt.nutanixCluster.GetPrismCentralTrustBundle()
+			assert.Equal(t, tt.expectedTrustBundle, trustBundle)
+		})
+	}
+}

controllers/nutanixcluster_controller.go

@@ -17,6 +17,7 @@ limitations under the License.
 package controllers
 
 import (
+	"cmp"
 	"context"
 	"fmt"
 	"time"
@@ -178,6 +179,11 @@ func (r *NutanixClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 	conditions.MarkTrue(cluster, infrav1.CredentialRefSecretOwnerSetCondition)
 
+	if err := r.reconcileTrustBundleRef(ctx, cluster); err != nil {
+		log.Error(err, fmt.Sprintf("error occurred while reconciling trust bundle ref for cluster %s", capiCluster.Name))
+		return reconcile.Result{}, err
+	}
+
 	v3Client, err := getPrismCentralClientForCluster(ctx, cluster, r.SecretInformer, r.ConfigMapInformer)
 	if err != nil {
 		log.Error(err, "error occurred while fetching prism central client")
@@ -373,6 +379,50 @@ func (r *NutanixClusterReconciler) reconcileCredentialRefDelete(ctx context.Cont
 	return nil
 }
 
+func (r *NutanixClusterReconciler) reconcileTrustBundleRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
+	log := ctrl.LoggerFrom(ctx)
+	trustBundleRef := nutanixCluster.GetPrismCentralTrustBundle()
+	if trustBundleRef == nil {
+		log.Info(fmt.Sprintf("trust bundle ref is nil for cluster %s", nutanixCluster.Name))
+		return nil
+	}
+
+	// get the trust bundle configmap
+	configMap := &corev1.ConfigMap{}
+	configMapKey := client.ObjectKey{
+		Namespace: cmp.Or(trustBundleRef.Namespace, nutanixCluster.Namespace),
+		Name:      trustBundleRef.Name,
+	}
+	if err := r.Client.Get(ctx, configMapKey, configMap); err != nil {
+		log.Error(err, "error occurred while fetching trust bundle configmap", "nutanixCluster", nutanixCluster.Name)
+		conditions.MarkFalse(nutanixCluster, infrav1.TrustBundleSecretOwnerSetCondition, infrav1.TrustBundleSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
+		return err
+	}
+
+	if !capiutil.IsOwnedByObject(configMap, nutanixCluster) {
+		// Check if another nutanixCluster already has set ownerRef. Secret can only be owned by one nutanixCluster object
+		if capiutil.HasOwner(configMap.OwnerReferences, infrav1.GroupVersion.String(), []string{nutanixCluster.Kind}) {
+			return fmt.Errorf("configmap %s/%s already owned by another nutanixCluster object", configMap.Namespace, configMap.Name)
+		}
+
+		configMap.OwnerReferences = capiutil.EnsureOwnerRef(configMap.OwnerReferences, metav1.OwnerReference{
+			APIVersion: infrav1.GroupVersion.String(),
+			Kind:       nutanixCluster.Kind,
+			UID:        nutanixCluster.UID,
+			Name:       nutanixCluster.Name,
+		})
+	}
+
+	if err := r.Client.Update(ctx, configMap); err != nil {
+		log.Error(err, "error occurred while updating trust bundle configmap", "nutanixCluster", nutanixCluster)
+		conditions.MarkFalse(nutanixCluster, infrav1.TrustBundleSecretOwnerSetCondition, infrav1.TrustBundleSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
+		return err
+	}
+
+	conditions.MarkTrue(nutanixCluster, infrav1.TrustBundleSecretOwnerSetCondition)
+	return nil
+}
+
 func (r *NutanixClusterReconciler) reconcileCredentialRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
 	log := ctrl.LoggerFrom(ctx)
 	credentialRef, err := getPrismCentralCredentialRefForCluster(nutanixCluster)