Skip to content

Commit

Permalink
Add Clusterctl Move E2E test
Browse files Browse the repository at this point in the history
Also set ownerReference on trust bundle configmaps so CMs get moved
when clusterctl move happens.
  • Loading branch information
thunderboltsid committed May 29, 2024
1 parent 0fdeb83 commit abcb42a
Show file tree
Hide file tree
Showing 10 changed files with 333 additions and 40 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
*.dylib
bin
testbin/*
*.tmp

# Test binary, build with `go test -c`
*.test
Expand Down
58 changes: 22 additions & 36 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,17 @@ GOTEST=$(GOCMD) test
GOGET=$(GOCMD) get
GOTOOL=$(GOCMD) tool
EXPORT_RESULT?=false # for CI please set EXPORT_RESULT to true
# Image URL to use all building/pushing image targets
IMG ?= ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest

GIT_COMMIT_HASH=$(shell git rev-parse HEAD)
LOCAL_IMAGE_REGISTRY ?= localhost:5000
IMG_REPO=${LOCAL_IMAGE_REGISTRY}/cluster-api-provider-nutanix
IMG_TAG=e2e-${GIT_COMMIT_HASH}
MANAGER_IMAGE=${IMG_REPO}:${IMG_TAG}

# Extract base and tag from IMG
IMG_REPO ?= $(word 1,$(subst :, ,${IMG}))
IMG_TAG ?= $(word 2,$(subst :, ,${IMG}))
LOCAL_PROVIDER_VERSION ?= ${IMG_TAG}
ifeq (${IMG_TAG},)
IMG_TAG := latest
endif

ifeq (${LOCAL_PROVIDER_VERSION},latest)
ifeq (${LOCAL_PROVIDER_VERSION},${IMG_TAG})
# TODO(release-blocker): Change this versions after release when required here and in e2e config (test/e2e/config/nutanix.yaml)
LOCAL_PROVIDER_VERSION := v1.4.99
endif
Expand Down Expand Up @@ -62,21 +61,6 @@ else
GOBIN=$(shell go env GOBIN)
endif

# Get latest git hash
GIT_COMMIT_HASH=$(shell git rev-parse HEAD)

# Get the local image registry required for clusterctl upgrade tests
LOCAL_IMAGE_REGISTRY ?= localhost:5000

ifeq (${MAKECMDGOALS},test-e2e-clusterctl-upgrade)
IMG_TAG=e2e-${GIT_COMMIT_HASH}
IMG_REPO=${LOCAL_IMAGE_REGISTRY}/controller
endif

ifeq (${MAKECMDGOALS},docker-build-e2e)
IMG_TAG=e2e-${GIT_COMMIT_HASH}
endif

# Setting SHELL to bash allows bash commands to be executed by recipes.
# This is a requirement for 'setup-envtest.sh' in the test target.
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
Expand All @@ -94,10 +78,11 @@ ifneq ($(LABEL_FILTERS),)
LABEL_FILTER_ARGS := "$(LABEL_FILTER_ARGS) && $(LABEL_FILTERS)"
endif
JUNIT_REPORT_FILE ?= "junit.e2e_suite.1.xml"
GINKGO_SKIP ?= "clusterctl-Upgrade"
GINKGO_SKIP ?= ""
GINKGO_FOCUS ?= ""
GINKGO_NODES ?= 1
E2E_CONF_FILE ?= ${E2E_DIR}/config/nutanix.yaml
E2E_CONF_FILE_TMP = ${E2E_CONF_FILE}.tmp
ARTIFACTS ?= ${REPO_ROOT}/_artifacts
SKIP_RESOURCE_CLEANUP ?= false
USE_EXISTING_CLUSTER ?= false
Expand Down Expand Up @@ -287,7 +272,7 @@ cluster-templates: ## Generate cluster templates for all flavors
docker-build-e2e: ## Build docker image with the manager with e2e tag.
echo "Git commit hash: ${GIT_COMMIT_HASH}"
KO_DOCKER_REPO=ko.local GOFLAGS="-ldflags=-X=main.gitCommitHash=${GIT_COMMIT_HASH}" ko build -B --platform=${PLATFORMS_E2E} -t ${IMG_TAG} .
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:e2e
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}

.PHONY: prepare-local-clusterctl
prepare-local-clusterctl: manifests cluster-templates ## Prepare overide file for local clusterctl.
Expand Down Expand Up @@ -332,6 +317,10 @@ template-test: cluster-templates ## Run the template tests

.PHONY: test-e2e
test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
echo "Image tag for E2E test is ${IMG_TAG}"
MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
docker push ${IMG_REPO}:${IMG_TAG}
mkdir -p $(ARTIFACTS)
NUTANIX_LOG_LEVEL=debug ginkgo -v \
--trace \
Expand All @@ -348,12 +337,16 @@ test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en
--always-emit-ginkgo-writer \
$(GINKGO_ARGS) ./test/e2e -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE)" \
-e2e.config="$(E2E_CONF_FILE_TMP)" \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)

.PHONY: test-e2e-no-kubeproxy
test-e2e-no-kubeproxy: docker-build-e2e cluster-e2e-templates-no-kubeproxy cluster-templates ## Run the end-to-end tests without kubeproxy
echo "Image tag for E2E test is ${IMG_TAG}"
MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
docker push ${IMG_REPO}:${IMG_TAG}
mkdir -p $(ARTIFACTS)
NUTANIX_LOG_LEVEL=debug ginkgo -v \
--trace \
Expand Down Expand Up @@ -385,15 +378,15 @@ list-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en

.PHONY: test-e2e-calico
test-e2e-calico:
CNI=$(CNI_PATH_CALICO) $(MAKE) test-e2e
CNI=$(CNI_PATH_CALICO) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e

.PHONY: test-e2e-flannel
test-e2e-flannel:
CNI=$(CNI_PATH_FLANNEL) $(MAKE) test-e2e
CNI=$(CNI_PATH_FLANNEL) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e

.PHONY: test-e2e-cilium
test-e2e-cilium:
CNI=$(CNI_PATH_CILIUM) $(MAKE) test-e2e
CNI=$(CNI_PATH_CILIUM) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e

.PHONY: test-e2e-cilium-no-kubeproxy
test-e2e-cilium-no-kubeproxy:
Expand All @@ -402,13 +395,6 @@ test-e2e-cilium-no-kubeproxy:
.PHONY: test-e2e-all-cni
test-e2e-all-cni: test-e2e test-e2e-calico test-e2e-flannel test-e2e-cilium test-e2e-cilium-no-kubeproxy

.PHONY: test-e2e-clusterctl-upgrade
test-e2e-clusterctl-upgrade: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
echo "Image tag for E2E test is ${IMG_TAG}"
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
docker push ${IMG_REPO}:${IMG_TAG}
GINKGO_SKIP="" GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e-calico

##@ Lint and Verify

.PHONY: lint
Expand Down
4 changes: 3 additions & 1 deletion api/v1beta1/conditions.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,7 @@ const (
// CredentialRefSecretOwnerSetCondition shows the status of setting the Owner
CredentialRefSecretOwnerSetCondition capiv1.ConditionType = "CredentialRefSecretOwnerSet"

CredentialRefSecretOwnerSetFailed = "CredentialRefSecretOwnerSetFailed"
CredentialRefSecretOwnerSetFailed = "CredentialRefSecretOwnerSetFailed"
TrustBundleSecretOwnerSetCondition = "TrustBundleSecretOwnerSet"
TrustBundleSecretOwnerSetFailed = "TrustBundleSecretOwnerSetFailed"
)
12 changes: 12 additions & 0 deletions api/v1beta1/nutanixcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,18 @@ func (ncl *NutanixCluster) GetPrismCentralCredentialRef() (*credentialTypes.Nuta
return prismCentralInfo.CredentialRef, nil
}

// GetPrismCentralTrustBundle returns the trust bundle reference for the Nutanix Prism Central.
func (ncl *NutanixCluster) GetPrismCentralTrustBundle() *credentialTypes.NutanixTrustBundleReference {
prismCentralInfo := ncl.Spec.PrismCentral
if prismCentralInfo == nil ||
prismCentralInfo.AdditionalTrustBundle == nil ||
prismCentralInfo.AdditionalTrustBundle.Kind == credentialTypes.NutanixTrustBundleKindString {
return nil
}

return prismCentralInfo.AdditionalTrustBundle
}

// GetNamespacedName returns the namespaced name of the NutanixCluster.
func (ncl *NutanixCluster) GetNamespacedName() string {
namespace := cmp.Or(ncl.Namespace, corev1.NamespaceDefault)
Expand Down
65 changes: 65 additions & 0 deletions api/v1beta1/nutanixcluster_types_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,3 +148,68 @@ func TestGetNamespacedName(t *testing.T) {
})
}
}

func TestGetPrismCentralTrustBundle(t *testing.T) {
t.Parallel()
tests := []struct {
name string
nutanixCluster *NutanixCluster
expectedTrustBundle *credentials.NutanixTrustBundleReference
}{
{
name: "PrismCentral and AdditionalTrustBundle are nil",
nutanixCluster: &NutanixCluster{
Spec: NutanixClusterSpec{},
},
expectedTrustBundle: nil,
},
{
name: "AdditionalTrustBundle is nil",
nutanixCluster: &NutanixCluster{
Spec: NutanixClusterSpec{
PrismCentral: &credentials.NutanixPrismEndpoint{},
},
},
expectedTrustBundle: nil,
},
{
name: "AdditionalTrustBundle Kind is NutanixTrustBundleKindString",
nutanixCluster: &NutanixCluster{
Spec: NutanixClusterSpec{
PrismCentral: &credentials.NutanixPrismEndpoint{
AdditionalTrustBundle: &credentials.NutanixTrustBundleReference{
Kind: credentials.NutanixTrustBundleKindString,
},
},
},
},
expectedTrustBundle: nil,
},
{
name: "AdditionalTrustBundle is not nil and Kind is not NutanixTrustBundleKindString",
nutanixCluster: &NutanixCluster{
Spec: NutanixClusterSpec{
PrismCentral: &credentials.NutanixPrismEndpoint{
AdditionalTrustBundle: &credentials.NutanixTrustBundleReference{
Kind: credentials.NutanixTrustBundleKindConfigMap,
Name: "test",
Namespace: "test-namespace",
},
},
},
},
expectedTrustBundle: &credentials.NutanixTrustBundleReference{
Kind: credentials.NutanixTrustBundleKindConfigMap,
Name: "test",
Namespace: "test-namespace",
},
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
trustBundle := tt.nutanixCluster.GetPrismCentralTrustBundle()
assert.Equal(t, tt.expectedTrustBundle, trustBundle)
})
}
}
50 changes: 50 additions & 0 deletions controllers/nutanixcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package controllers

import (
"cmp"
"context"
"fmt"
"time"
Expand Down Expand Up @@ -178,6 +179,11 @@ func (r *NutanixClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque
}
conditions.MarkTrue(cluster, infrav1.CredentialRefSecretOwnerSetCondition)

if err := r.reconcileTrustBundleRef(ctx, cluster); err != nil {
log.Error(err, fmt.Sprintf("error occurred while reconciling trust bundle ref for cluster %s", capiCluster.Name))
return reconcile.Result{}, err

Check warning on line 184 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L182-L184

Added lines #L182 - L184 were not covered by tests
}

v3Client, err := getPrismCentralClientForCluster(ctx, cluster, r.SecretInformer, r.ConfigMapInformer)
if err != nil {
log.Error(err, "error occurred while fetching prism central client")
Expand Down Expand Up @@ -373,6 +379,50 @@ func (r *NutanixClusterReconciler) reconcileCredentialRefDelete(ctx context.Cont
return nil
}

func (r *NutanixClusterReconciler) reconcileTrustBundleRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
log := ctrl.LoggerFrom(ctx)
trustBundleRef := nutanixCluster.GetPrismCentralTrustBundle()
if trustBundleRef == nil {
log.Info(fmt.Sprintf("trust bundle ref is nil for cluster %s", nutanixCluster.Name))
return nil
}

// get the trust bundle configmap
configMap := &corev1.ConfigMap{}
configMapKey := client.ObjectKey{
Namespace: cmp.Or(trustBundleRef.Namespace, nutanixCluster.Namespace),
Name: trustBundleRef.Name,
}
if err := r.Client.Get(ctx, configMapKey, configMap); err != nil {
log.Error(err, "error occurred while fetching trust bundle configmap", "nutanixCluster", nutanixCluster.Name)
conditions.MarkFalse(nutanixCluster, infrav1.TrustBundleSecretOwnerSetCondition, infrav1.TrustBundleSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
return err
}

if !capiutil.IsOwnedByObject(configMap, nutanixCluster) {
// Check if another nutanixCluster already has set ownerRef. Secret can only be owned by one nutanixCluster object
if capiutil.HasOwner(configMap.OwnerReferences, infrav1.GroupVersion.String(), []string{nutanixCluster.Kind}) {
return fmt.Errorf("configmap %s/%s already owned by another nutanixCluster object", configMap.Namespace, configMap.Name)

Check warning on line 405 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L405

Added line #L405 was not covered by tests
}

configMap.OwnerReferences = capiutil.EnsureOwnerRef(configMap.OwnerReferences, metav1.OwnerReference{
APIVersion: infrav1.GroupVersion.String(),
Kind: nutanixCluster.Kind,
UID: nutanixCluster.UID,
Name: nutanixCluster.Name,
})
}

if err := r.Client.Update(ctx, configMap); err != nil {
log.Error(err, "error occurred while updating trust bundle configmap", "nutanixCluster", nutanixCluster)
conditions.MarkFalse(nutanixCluster, infrav1.TrustBundleSecretOwnerSetCondition, infrav1.TrustBundleSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
return err
}

conditions.MarkTrue(nutanixCluster, infrav1.TrustBundleSecretOwnerSetCondition)
return nil
}

func (r *NutanixClusterReconciler) reconcileCredentialRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
log := ctrl.LoggerFrom(ctx)
credentialRef, err := getPrismCentralCredentialRefForCluster(nutanixCluster)
Expand Down
Loading

0 comments on commit abcb42a

Please sign in to comment.