Skip to content

Commit

Permalink
Add Clusterctl Move E2E test
Browse files Browse the repository at this point in the history
Also set ownerReference on trust bundle configmaps so CMs get moved
when clusterctl move happens.
  • Loading branch information
thunderboltsid committed May 29, 2024
1 parent 0fdeb83 commit 3e89af5
Show file tree
Hide file tree
Showing 9 changed files with 137 additions and 44 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
*.dylib
bin
testbin/*
*.tmp

# Test binary, build with `go test -c`
*.test
Expand Down
58 changes: 22 additions & 36 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,17 @@ GOTEST=$(GOCMD) test
GOGET=$(GOCMD) get
GOTOOL=$(GOCMD) tool
EXPORT_RESULT?=false # for CI please set EXPORT_RESULT to true
# Image URL to use all building/pushing image targets
IMG ?= ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest

GIT_COMMIT_HASH=$(shell git rev-parse HEAD)
LOCAL_IMAGE_REGISTRY ?= localhost:5000
IMG_REPO=${LOCAL_IMAGE_REGISTRY}/cluster-api-provider-nutanix
IMG_TAG=e2e-${GIT_COMMIT_HASH}
MANAGER_IMAGE=${IMG_REPO}:${IMG_TAG}

# Extract base and tag from IMG
IMG_REPO ?= $(word 1,$(subst :, ,${IMG}))
IMG_TAG ?= $(word 2,$(subst :, ,${IMG}))
LOCAL_PROVIDER_VERSION ?= ${IMG_TAG}
ifeq (${IMG_TAG},)
IMG_TAG := latest
endif

ifeq (${LOCAL_PROVIDER_VERSION},latest)
ifeq (${LOCAL_PROVIDER_VERSION},${IMG_TAG})
# TODO(release-blocker): Change this versions after release when required here and in e2e config (test/e2e/config/nutanix.yaml)
LOCAL_PROVIDER_VERSION := v1.4.99
endif
Expand Down Expand Up @@ -62,21 +61,6 @@ else
GOBIN=$(shell go env GOBIN)
endif

# Get latest git hash
GIT_COMMIT_HASH=$(shell git rev-parse HEAD)

# Get the local image registry required for clusterctl upgrade tests
LOCAL_IMAGE_REGISTRY ?= localhost:5000

ifeq (${MAKECMDGOALS},test-e2e-clusterctl-upgrade)
IMG_TAG=e2e-${GIT_COMMIT_HASH}
IMG_REPO=${LOCAL_IMAGE_REGISTRY}/controller
endif

ifeq (${MAKECMDGOALS},docker-build-e2e)
IMG_TAG=e2e-${GIT_COMMIT_HASH}
endif

# Setting SHELL to bash allows bash commands to be executed by recipes.
# This is a requirement for 'setup-envtest.sh' in the test target.
# Options are set to exit when a recipe line exits non-zero or a piped command fails.
Expand All @@ -94,10 +78,11 @@ ifneq ($(LABEL_FILTERS),)
LABEL_FILTER_ARGS := "$(LABEL_FILTER_ARGS) && $(LABEL_FILTERS)"
endif
JUNIT_REPORT_FILE ?= "junit.e2e_suite.1.xml"
GINKGO_SKIP ?= "clusterctl-Upgrade"
GINKGO_SKIP ?= ""
GINKGO_FOCUS ?= ""
GINKGO_NODES ?= 1
E2E_CONF_FILE ?= ${E2E_DIR}/config/nutanix.yaml
E2E_CONF_FILE_TMP = ${E2E_CONF_FILE}.tmp
ARTIFACTS ?= ${REPO_ROOT}/_artifacts
SKIP_RESOURCE_CLEANUP ?= false
USE_EXISTING_CLUSTER ?= false
Expand Down Expand Up @@ -287,7 +272,7 @@ cluster-templates: ## Generate cluster templates for all flavors
docker-build-e2e: ## Build docker image with the manager with e2e tag.
echo "Git commit hash: ${GIT_COMMIT_HASH}"
KO_DOCKER_REPO=ko.local GOFLAGS="-ldflags=-X=main.gitCommitHash=${GIT_COMMIT_HASH}" ko build -B --platform=${PLATFORMS_E2E} -t ${IMG_TAG} .
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:e2e
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}

.PHONY: prepare-local-clusterctl
prepare-local-clusterctl: manifests cluster-templates ## Prepare overide file for local clusterctl.
Expand Down Expand Up @@ -332,6 +317,10 @@ template-test: cluster-templates ## Run the template tests

.PHONY: test-e2e
test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
echo "Image tag for E2E test is ${IMG_TAG}"
MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
docker push ${IMG_REPO}:${IMG_TAG}
mkdir -p $(ARTIFACTS)
NUTANIX_LOG_LEVEL=debug ginkgo -v \
--trace \
Expand All @@ -348,12 +337,16 @@ test-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en
--always-emit-ginkgo-writer \
$(GINKGO_ARGS) ./test/e2e -- \
-e2e.artifacts-folder="$(ARTIFACTS)" \
-e2e.config="$(E2E_CONF_FILE)" \
-e2e.config="$(E2E_CONF_FILE_TMP)" \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER)

.PHONY: test-e2e-no-kubeproxy
test-e2e-no-kubeproxy: docker-build-e2e cluster-e2e-templates-no-kubeproxy cluster-templates ## Run the end-to-end tests without kubeproxy
echo "Image tag for E2E test is ${IMG_TAG}"
MANAGER_IMAGE=$(MANAGER_IMAGE) envsubst < ${E2E_CONF_FILE} > ${E2E_CONF_FILE_TMP}
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
docker push ${IMG_REPO}:${IMG_TAG}
mkdir -p $(ARTIFACTS)
NUTANIX_LOG_LEVEL=debug ginkgo -v \
--trace \
Expand Down Expand Up @@ -385,15 +378,15 @@ list-e2e: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the en

.PHONY: test-e2e-calico
test-e2e-calico:
CNI=$(CNI_PATH_CALICO) $(MAKE) test-e2e
CNI=$(CNI_PATH_CALICO) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e

.PHONY: test-e2e-flannel
test-e2e-flannel:
CNI=$(CNI_PATH_FLANNEL) $(MAKE) test-e2e
CNI=$(CNI_PATH_FLANNEL) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e

.PHONY: test-e2e-cilium
test-e2e-cilium:
CNI=$(CNI_PATH_CILIUM) $(MAKE) test-e2e
CNI=$(CNI_PATH_CILIUM) GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e

.PHONY: test-e2e-cilium-no-kubeproxy
test-e2e-cilium-no-kubeproxy:
Expand All @@ -402,13 +395,6 @@ test-e2e-cilium-no-kubeproxy:
.PHONY: test-e2e-all-cni
test-e2e-all-cni: test-e2e test-e2e-calico test-e2e-flannel test-e2e-cilium test-e2e-cilium-no-kubeproxy

.PHONY: test-e2e-clusterctl-upgrade
test-e2e-clusterctl-upgrade: docker-build-e2e cluster-e2e-templates cluster-templates ## Run the end-to-end tests
echo "Image tag for E2E test is ${IMG_TAG}"
docker tag ko.local/cluster-api-provider-nutanix:${IMG_TAG} ${IMG_REPO}:${IMG_TAG}
docker push ${IMG_REPO}:${IMG_TAG}
GINKGO_SKIP="" GIT_COMMIT="${GIT_COMMIT_HASH}" $(MAKE) test-e2e-calico

##@ Lint and Verify

.PHONY: lint
Expand Down
4 changes: 3 additions & 1 deletion api/v1beta1/conditions.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,5 +74,7 @@ const (
// CredentialRefSecretOwnerSetCondition shows the status of setting the Owner
CredentialRefSecretOwnerSetCondition capiv1.ConditionType = "CredentialRefSecretOwnerSet"

CredentialRefSecretOwnerSetFailed = "CredentialRefSecretOwnerSetFailed"
CredentialRefSecretOwnerSetFailed = "CredentialRefSecretOwnerSetFailed"
TrustBundleSecretOwnerSetCondition = "TrustBundleSecretOwnerSet"
TrustBundleSecretOwnerSetFailed = "TrustBundleSecretOwnerSetFailed"
)
11 changes: 11 additions & 0 deletions api/v1beta1/nutanixcluster_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,17 @@ func (ncl *NutanixCluster) GetPrismCentralCredentialRef() (*credentialTypes.Nuta
return prismCentralInfo.CredentialRef, nil
}

func (ncl *NutanixCluster) GetPrismCentralTrustBundle() *credentialTypes.NutanixTrustBundleReference {
prismCentralInfo := ncl.Spec.PrismCentral
if prismCentralInfo == nil ||

Check warning on line 169 in api/v1beta1/nutanixcluster_types.go

View check run for this annotation

Codecov / codecov/patch

api/v1beta1/nutanixcluster_types.go#L168-L169

Added lines #L168 - L169 were not covered by tests
prismCentralInfo.AdditionalTrustBundle == nil ||
prismCentralInfo.AdditionalTrustBundle.Kind == credentialTypes.NutanixTrustBundleKindString {
return nil

Check warning on line 172 in api/v1beta1/nutanixcluster_types.go

View check run for this annotation

Codecov / codecov/patch

api/v1beta1/nutanixcluster_types.go#L172

Added line #L172 was not covered by tests
}

return prismCentralInfo.AdditionalTrustBundle

Check warning on line 175 in api/v1beta1/nutanixcluster_types.go

View check run for this annotation

Codecov / codecov/patch

api/v1beta1/nutanixcluster_types.go#L175

Added line #L175 was not covered by tests
}

// GetNamespacedName returns the namespaced name of the NutanixCluster.
func (ncl *NutanixCluster) GetNamespacedName() string {
namespace := cmp.Or(ncl.Namespace, corev1.NamespaceDefault)
Expand Down
6 changes: 2 additions & 4 deletions config/manager/kustomization.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,5 @@ configMapGenerator:
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

images:
- name: controller
newName: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller
newTag: latest
patches:
- path: manager_image_patch.yaml
12 changes: 12 additions & 0 deletions config/manager/manager_image_patch.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
spec:
template:
spec:
containers:
# Change the value of image field below to your controller image URL
- image: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest
name: manager
44 changes: 44 additions & 0 deletions controllers/nutanixcluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ limitations under the License.
package controllers

import (
"cmp"
"context"
"fmt"
"time"
Expand Down Expand Up @@ -178,6 +179,13 @@ func (r *NutanixClusterReconciler) Reconcile(ctx context.Context, req ctrl.Reque
}
conditions.MarkTrue(cluster, infrav1.CredentialRefSecretOwnerSetCondition)

if err := r.reconcileTrustBundleRef(ctx, cluster); err != nil {
log.Error(err, fmt.Sprintf("error occurred while reconciling trust bundle ref for cluster %s", capiCluster.Name))
conditions.MarkFalse(cluster, infrav1.TrustBundleSecretOwnerSetCondition, infrav1.TrustBundleSecretOwnerSetFailed, capiv1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err

Check warning on line 185 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L182-L185

Added lines #L182 - L185 were not covered by tests
}
conditions.MarkTrue(cluster, infrav1.TrustBundleSecretOwnerSetCondition)

Check warning on line 187 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L187

Added line #L187 was not covered by tests

v3Client, err := getPrismCentralClientForCluster(ctx, cluster, r.SecretInformer, r.ConfigMapInformer)
if err != nil {
log.Error(err, "error occurred while fetching prism central client")
Expand Down Expand Up @@ -373,6 +381,42 @@ func (r *NutanixClusterReconciler) reconcileCredentialRefDelete(ctx context.Cont
return nil
}

func (r *NutanixClusterReconciler) reconcileTrustBundleRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
log := ctrl.LoggerFrom(ctx)
trustBundleRef := nutanixCluster.GetPrismCentralTrustBundle()
if trustBundleRef == nil {
log.Info(fmt.Sprintf("trust bundle ref is nil for cluster %s", nutanixCluster.Name))
return nil

Check warning on line 389 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L385-L389

Added lines #L385 - L389 were not covered by tests
}

// get the trust bundle configmap
configMap := &corev1.ConfigMap{}
configMapKey := client.ObjectKey{

Check warning on line 394 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L393-L394

Added lines #L393 - L394 were not covered by tests
Namespace: cmp.Or(trustBundleRef.Namespace, nutanixCluster.Namespace),
Name: trustBundleRef.Name,
}
if err := r.Client.Get(ctx, configMapKey, configMap); err != nil {
log.Error(err, "error occurred while fetching trust bundle configmap", "nutanixCluster", nutanixCluster.Name)
return err

Check warning on line 400 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L398-L400

Added lines #L398 - L400 were not covered by tests
}

if !capiutil.IsOwnedByObject(configMap, nutanixCluster) {
configMap.OwnerReferences = capiutil.EnsureOwnerRef(configMap.OwnerReferences, metav1.OwnerReference{

Check warning on line 404 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L403-L404

Added lines #L403 - L404 were not covered by tests
APIVersion: infrav1.GroupVersion.String(),
Kind: nutanixCluster.Kind,
UID: nutanixCluster.UID,
Name: nutanixCluster.Name,
})
}

if err := r.Client.Update(ctx, configMap); err != nil {
log.Error(err, "error occurred while updating trust bundle configmap", "nutanixCluster", nutanixCluster)
return err

Check warning on line 414 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L412-L414

Added lines #L412 - L414 were not covered by tests
}

return nil

Check warning on line 417 in controllers/nutanixcluster_controller.go

View check run for this annotation

Codecov / codecov/patch

controllers/nutanixcluster_controller.go#L417

Added line #L417 was not covered by tests
}

func (r *NutanixClusterReconciler) reconcileCredentialRef(ctx context.Context, nutanixCluster *infrav1.NutanixCluster) error {
log := ctrl.LoggerFrom(ctx)
credentialRef, err := getPrismCentralCredentialRefForCluster(nutanixCluster)
Expand Down
39 changes: 39 additions & 0 deletions test/e2e/clusterctl_move_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
//go:build e2e

/*
Copyright 2024 Nutanix
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
. "github.com/onsi/ginkgo/v2"
capie2e "sigs.k8s.io/cluster-api/test/e2e"
)

var _ = Describe("[clusterctl-move] Create a Self-Hosted CAPX Cluster", Label("capx-feature-test", "clusterctl-move"), func() {
// Run the CAPI SelfHostedSpec test suite
// This test suite will create a self-hosted CAPX cluster
capie2e.SelfHostedSpec(ctx, func() capie2e.SelfHostedSpecInput {
return capie2e.SelfHostedSpecInput{
E2EConfig: e2eConfig,
ClusterctlConfigPath: clusterctlConfigPath,
BootstrapClusterProxy: bootstrapClusterProxy,
ArtifactFolder: artifactFolder,
SkipCleanup: skipCleanup,
SkipUpgrade: true,
}
})
})
6 changes: 3 additions & 3 deletions test/e2e/config/nutanix.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

images:
# Use local dev images built source tree;
- name: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:e2e
- name: ${MANAGER_IMAGE}
loadBehavior: mustLoad
# ## PLEASE KEEP THESE UP TO DATE WITH THE COMPONENTS
# Cluster API v1beta1 Preloads
Expand Down Expand Up @@ -193,10 +193,10 @@ providers:
value: "../../../config/default"
contract: v1beta1
replacements:
- old: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest
new: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:e2e
- old: "imagePullPolicy: Always"
new: "imagePullPolicy: IfNotPresent"
- old: "image: ghcr.io/nutanix-cloud-native/cluster-api-provider-nutanix/controller:latest"
new: "image: ${MANAGER_IMAGE}"
files:
- sourcePath: "../../../metadata.yaml"
- sourcePath: "../data/infrastructure-nutanix/v1beta1/cluster-template.yaml"
Expand Down

0 comments on commit 3e89af5

Please sign in to comment.