Merge pull request vmware-tanzu#7353 from kaovilai/e2e-updates

E2E usability updates

.github/workflows/e2e-test-kind.yaml

@@ -67,6 +67,16 @@ jobs:
           - 1.23.6
           - 1.24.0
           - 1.25.3
+        focus:
+          # tests to focus on, use `|` to concatenate multiple regexes to run on the same job
+          # ordered according to e2e_suite_test.go order
+          - Basic\]\[ClusterResource
+          - ResourceFiltering
+          - ResourceModifier|Backups|PrivilegesMgmt\]\[SSR
+          - Schedule\]\[OrderedResources
+          - NamespaceMapping\]\[Single\]\[Restic|NamespaceMapping\]\[Multiple\]\[Restic
+          - Basic\]\[Nodeport
+          - Basic\]\[StorageClass
       fail-fast: false
     steps:
       - name: Set up Go
@@ -123,7 +133,8 @@ jobs:
               CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \
               ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \
               ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \
-              GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \
+              GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \
+              GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \
               make -C test/e2e run
         timeout-minutes: 30
       - name: Upload debug bundle

Makefile

@@ -108,6 +108,7 @@ platform_temp = $(subst -, ,$(ARCH))
 GOOS = $(word 1, $(platform_temp))
 GOARCH = $(word 2, $(platform_temp))
 GOPROXY ?= https://proxy.golang.org
+GOBIN=$$(pwd)/.go/bin
 
 # If you want to build all binaries, see the 'all-build' rule.
 # If you want to build all containers, see the 'all-containers' rule.
@@ -129,6 +130,7 @@ local: build-dirs
 # Add DEBUG=1 to enable debug locally
 	GOOS=$(GOOS) \
 	GOARCH=$(GOARCH) \
+	GOBIN=$(GOBIN) \
 	VERSION=$(VERSION) \
 	REGISTRY=$(REGISTRY) \
 	PKG=$(PKG) \
@@ -145,6 +147,7 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs
 	$(MAKE) shell CMD="-c '\
 		GOOS=$(GOOS) \
 		GOARCH=$(GOARCH) \
+		GOBIN=$(GOBIN) \
 		VERSION=$(VERSION) \
 		REGISTRY=$(REGISTRY) \
 		PKG=$(PKG) \

hack/build.sh

@@ -32,6 +32,10 @@ if [[ -z "${GOOS}" ]]; then
     echo "GOOS must be set"
     exit 1
 fi
+if [[ -z "${GOBIN}" ]]; then
+    echo "GOBIN must be set"
+    exit 1
+fi
 if [[ -z "${GOARCH}" ]]; then
     echo "GOARCH must be set"
     exit 1

test/e2e/Makefile

@@ -41,8 +41,10 @@ help:  ## Display this help
 
 TOOLS_DIR := $(REPO_ROOT)/hack/tools
 BIN_DIR := bin
+# Try to not modify PATH if possible
+GOBIN := $(REPO_ROOT)/.go/bin
 TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
-GINKGO := $(shell go env GOPATH)/bin/ginkgo
+GINKGO := $(GOBIN)/ginkgo
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
 GINKGO_FOCUS ?=
@@ -113,9 +115,13 @@ STANDBY_CLUSTER_NAME ?=
 EKS_POLICY_ARN ?=
 
 
+# Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo
-ginkgo: # Make sure ginkgo is in $GOPATH/bin
-	go install github.com/onsi/ginkgo/[email protected]
+ginkgo: ${GOBIN}/ginkgo
+
+# This target does not run if ginkgo is already in $GOBIN
+${GOBIN}/ginkgo:
+	GOBIN=${GOBIN} go install github.com/onsi/ginkgo/[email protected]
 
 .PHONY: run
 run: ginkgo

test/e2e/README.md

@@ -66,7 +66,7 @@ the object-store-provider to be specified.
 1. `-features`: Comma-separated list of features to enable for this Velero process.
 1. `-registry-credential-file`: File containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. This credential will be loaded in Velero server pod to help on Docker Hub rate limit issue.
 1. `-kibishii-directory`: The file directory or URL path to install Kibishii. It's configurable in case the default path is not accessible for your own test environment.
-1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action.
+1. `-debug-e2e-test`: <true/false> A Switch for enable or disable test data cleaning action.
 1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional.
 1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server.
 1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test.
@@ -101,7 +101,7 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `VELERO_NAMESPACE `: the `-velero-namespace`. Optional.
 1. `PLUGINS `: the `-plugins`. Optional.
 1. `BSL_PREFIX`: `-prefix`. Optional.
-1. `BSL_CONFIG`: `-bsl-config`. Optional.
+1. `BSL_CONFIG`: `-bsl-config`. Optional. Example: BSL_CONFIG="region=us-east-1". May be required for some object store provider
 1. `VSL_CONFIG`: `-vsl-config`. Optional.
 1. `UPGRADE_FROM_VELERO_CLI `: `-upgrade-from-velero-cli`. Optional.
 1. `UPGRADE_FROM_VELERO_VERSION `: `-upgrade-from-velero-version`. Optional.
@@ -136,9 +136,18 @@ Below is a mapping between `make` variables to E2E configuration flags.
 Basic examples:
 
 1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider:
+
+    Start kind cluster
+    ```bash
+    kind create cluster
+    ```
     ```bash
     BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws make test-e2e
     ```
+    Stop kind cluster
+    ```bash
+    kind delete cluster
+    ```
 1. Run Velero tests in an AWS cluster:
     ```bash
     BSL_PREFIX=<PREFIX_UNDER_BUCKET> BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=aws make test-e2e
@@ -303,4 +312,8 @@ The TestFunc function concatenate all the flows in a test.
 It will reduce the frequency of velero installation by installing and checking Velero with the global Velero. It's Need to explicit reinstall Velero if the case has specicial configuration, such as API Group test case we need to enable feature EnableCSI.
 
 ### Tips
-Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. 
+Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. 
+
+#### Troubleshooting
+
+If velero log shows `level=error msg="Failed to get bucket region, bucket: xbucket, error: operation error S3: HeadBucket, failed to resolve service endpoint, endpoint rule error, A region must be set when sending requests to S3." backup-storage-location=velero/default cmd=/plugins/velero-plugin-for-aws controller=backup-storage-location logSource="/go/src/velero-plugin-for-aws/velero-plugin-for-aws/object_store.go:136" pluginName=velero-plugin-for-aws`, it means you need to set `BSL_CONFIG` to include `region=<region>`.

test/e2e/backup/backup.go

@@ -74,8 +74,12 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 		veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir
 		veleroCfg.UseVolumeSnapshots = useVolumeSnapshots
 		veleroCfg.UseNodeAgent = !useVolumeSnapshots
-		if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
-			Skip("Volume snapshots not supported on kind")
+		if veleroCfg.CloudProvider == "kind" {
+			Skip("Volume snapshots plugin and File System Backups are not supported on kind")
+			// on kind cluster snapshots are not supported since there is no velero snapshot plugin for kind volumes.
+			// and PodVolumeBackups are not supported because PVB creation gets skipped for hostpath volumes, which are the only
+			// volumes created on kind clusters using the default storage class and provisioner (provisioner: rancher.io/local-path)
+			// This test suite checks for volume snapshots and PVBs generated from FileSystemBackups, so skip it on kind clusters
 		}
 		// [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers.
 		//         For vSphere CSI, after you create a PV specifying an existing volume handle, CSI

test/e2e/backups/deletion.go

@@ -86,6 +86,9 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 // runUpgradeTests runs upgrade test on the provider by kibishii.
 func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string,
 	useVolumeSnapshots bool, kibishiiDirectory string) error {
+	if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" {
+		Skip("Volume snapshots not supported on kind")
+	}
 	oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60)
 	defer ctxCancel()
 	veleroCLI := veleroCfg.VeleroCLI
@@ -111,7 +114,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil {
 		return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest)
 	}
-	err := ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
+	err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1)
 	if err != nil {
 		return err
 	}
@@ -139,7 +142,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 			return errors.Wrapf(err, "Error waiting for uploads to complete")
 		}
 	}
-	err = ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
+	err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
 	if err != nil {
 		return err
 	}
@@ -167,7 +170,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		}
 	}
 
-	err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
+	err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5)
 	if err != nil {
 		return err
 	}
@@ -194,12 +197,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam
 		})
 	})
 
-	err = DeleteObjectsInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
+	err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix)
 	if err != nil {
 		return err
 	}
 
-	err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
+	err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1)
 	if err != nil {
 		return err
 	}

test/e2e/backups/sync_backups.go

@@ -148,13 +148,13 @@ func BackupsSyncTest() {
 		})
 
 		By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() {
-			err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+			err = DeleteObjectsInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix)
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
 		})
 
 		By(fmt.Sprintf("Check %s backup files in object store is deleted", test.backupName), func() {
-			err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
+			err = ObjectsShouldNotBeInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket,
 				VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 1)
 			Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err))
 		})

test/e2e/backups/ttl.go

@@ -155,7 +155,7 @@ func TTLTest() {
 		})
 
 		By("Associated Restores should be created", func() {
-			Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
 				RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object")
@@ -179,7 +179,7 @@ func TTLTest() {
 		})
 
 		By("Backup file from cloud object storage should be deleted", func() {
-			Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName,
 				BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
@@ -194,7 +194,7 @@ func TTLTest() {
 		})
 
 		By("Associated Restores should be deleted", func() {
-			Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider,
+			Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider,
 				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
 				veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName,
 				RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object")

test/e2e/basic/namespace-mapping.go

@@ -38,6 +38,9 @@ func (n *NamespaceMapping) Init() error {
 	n.VeleroCfg.UseVolumeSnapshots = n.UseVolumeSnapshots
 	n.VeleroCfg.UseNodeAgent = !n.UseVolumeSnapshots
 	n.kibishiiData = &KibishiiData{Levels: 2, DirsPerLevel: 10, FilesPerLevel: 10, FileLength: 1024, BlockSize: 1024, PassNum: 0, ExpectedNodes: 2}
+	if n.VeleroCfg.CloudProvider == "kind" {
+		n.kibishiiData = &KibishiiData{Levels: 0, DirsPerLevel: 0, FilesPerLevel: 0, FileLength: 0, BlockSize: 0, PassNum: 0, ExpectedNodes: 2}
+	}
 	backupType := "restic"
 	if n.UseVolumeSnapshots {
 		backupType = "snapshot"
@@ -67,7 +70,11 @@ func (n *NamespaceMapping) Init() error {
 		"create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName,
 		"--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait",
 	}
-	if n.UseVolumeSnapshots {
+	if VeleroCfg.CloudProvider == "kind" {
+		// don't test volume snapshotter or file system backup on kind
+		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")
+		n.UseVolumeSnapshots = false
+	} else if n.UseVolumeSnapshots {
 		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes")
 	} else {
 		n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false")

test/e2e/bsl-mgmt/deletion.go

@@ -276,7 +276,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 				Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty())
 
 				By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() {
-					Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
+					Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
 						veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
 						backupName_1, BackupObjectsPrefix)).To(Succeed())
 				})
@@ -295,14 +295,14 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 			})
 
 			By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() {
-				Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile,
+				Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile,
 					veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig,
 					backupName_1, BackupObjectsPrefix)).To(Succeed())
 			})
 
 			// TODO: Choose additional BSL to be deleted as an new test case
 			// By(fmt.Sprintf("Backup %s should still exist in cloud object store", backupName_2), func() {
-			// 	Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.AdditionalBSLCredentials,
+			// 	Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.AdditionalBSLCredentials,
 			// 		veleroCfg.AdditionalBSLBucket, veleroCfg.AdditionalBSLPrefix, veleroCfg.AdditionalBSLConfig,
 			// 		backupName_2, BackupObjectsPrefix)).To(Succeed())
 			// })