Add E2E test for multiple credentials

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26
 	github.com/joho/godotenv v1.3.0
 	github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0
-	github.com/onsi/ginkgo v1.15.0
+	github.com/onsi/ginkgo v1.15.1
 	github.com/onsi/gomega v1.10.2
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.7.1

go.sum

@@ -419,6 +419,8 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
 github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
@@ -429,8 +431,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
-github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
-github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
+github.com/onsi/ginkgo v1.15.1 h1:DsXNrKujDlkMS9Rsxmd+Fg7S6Kc5lhE+qX8tY6laOxc=
+github.com/onsi/ginkgo v1.15.1/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=

test/e2e/Makefile

@@ -57,6 +57,13 @@ VSL_CONFIG ?=
 CLOUD_PROVIDER ?=
 OBJECT_STORE_PROVIDER ?=
 
+# Flags to create an additional BSL for multiple credentials tests
+ADDITIONAL_OBJECT_STORE_PROVIDER ?=
+ADDITIONAL_CREDS_FILE ?=
+ADDITIONAL_BSL_BUCKET ?=
+ADDITIONAL_BSL_PREFIX ?=
+ADDITIONAL_BSL_CONFIG ?=
+
 .PHONY:ginkgo
 ginkgo: # Make sure ginkgo is in $GOPATH/bin
 	go get github.com/onsi/ginkgo/ginkgo
@@ -78,7 +85,12 @@ run: ginkgo
 		-bsl-config=$(BSL_CONFIG) \
 		-vsl-config=$(VSL_CONFIG) \
 		-cloud-provider=$(CLOUD_PROVIDER) \
-		-object-store-provider="$(OBJECT_STORE_PROVIDER)"
+		-object-store-provider="$(OBJECT_STORE_PROVIDER)" \
+		-additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \
+		-additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \
+		-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
+		-additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
+		-additional-bsl-config=$(ADDITIONAL_BSL_CONFIG)
 
 build: ginkgo
 	mkdir -p $(OUTPUT_DIR)

test/e2e/README.md

@@ -41,6 +41,11 @@ the object-store-provider to be specified.
 1. `-vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional.
 1. `-velero-namespace`: Namespace to install velero in.  Optional, defaults to "velero".
 1. `-install-velero`: Specifies whether to install/uninstall velero for the tests.  Optional, defaults to "true".
+1. `-additional-bsl-object-store-provider`: Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.
+1. `-additional-bsl-bucket`: Name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.
+1. `-additional-bsl-prefix`: Prefix in the `additional-bsl-bucket`, under which all Velero data should be stored. Optional.
+1. `-additional-bsl-config`: Configuration to use for the additional backup storage location. Format is key1=value1,key2=value2. Optional.
+1. `-additional-bsl-credentials-file`: File containing credentials for the additional backup storage location. Required if testing multiple credentials support.
 
 These configurations or parameters are used to generate install options for Velero for each test suite.
 
@@ -62,6 +67,11 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `BSL_PREFIX`: `-prefix`. Optional.
 1. `BSL_CONFIG`: `-bsl-config`. Optional.
 1. `VSL_CONFIG`: `-vsl-config`. Optional.
+1. `ADDITIONAL_OBJECT_STORE_PROVIDER`: `-additional-bsl-object-store-provider`. Optional.
+1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-bucket`. Optional.
+1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-prefix`. Optional.
+1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-config`. Optional.
+1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-credentials-file`. Optional.
 
 For example, E2E tests can be run from Velero repository roots using the commands below:
 
@@ -82,6 +92,13 @@ For example, E2E tests can be run from Velero repository roots using the command
    ```bash
    BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=http://192.168.1.124:9000" BSL_PREFIX=veldat BSL_BUCKET=velero CREDS_FILE=~/go/src/github.com/vmware-tanzu/velero/frankie-secrets/credentials-minio PLUGIN_PROVIDER=aws VELERO_IMAGE=projects.registry.vmware.com/tanzu_migrator/velero-pr3133:0.0.5 GINKGO_FOCUS="API group versions" make test-e2e
    ```
+1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location:
+    ```bash
+    make test-e2e \
+      CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> BSL_PREFIX=<PREFIX_UNDER_BUCKET> CREDS_FILE=/path/to/aws-creds \
+      ADDITIONAL_OBJECT_STORE_PROVIDER=azure ADDITIONAL_BSL_BUCKET=<BUCKET_FOR_AZURE_BSL> ADDITIONAL_BSL_PREFIX=<PREFIX_UNDER_BUCKET> ADDITIONAL_BSL_CONFIG=<CONFIG_FOR_AZURE_BUCKET> ADDITIONAL_CREDS_FILE=/path/to/azure-creds
+    ```
+   Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container)
 
 ## Filtering tests
 

test/e2e/backup_test.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"context"
 	"flag"
+	"fmt"
 	"time"
 
 	"github.com/google/uuid"
@@ -49,14 +50,61 @@ var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for
 
 	})
 
-	Context("When kibishii is the sample workload", func() {
-		It("should be successfully backed up and restored", func() {
+	When("kibishii is the sample workload", func() {
+		It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
 			backupName = "backup-" + uuidgen.String()
 			restoreName = "restore-" + uuidgen.String()
 			// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
 			// KinD. So use the kind installation for Kibishii.
-			Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName)).To(Succeed(),
+			Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "")).To(Succeed(),
 				"Failed to successfully backup and restore Kibishii namespace")
 		})
+
+		It("should successfully back up and restore to multiple BackupStorageLocations with unique credentials", func() {
+			if additionalBslProvider == "" {
+				Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
+			}
+
+			if additionalBslBucket == "" {
+				Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
+			}
+
+			if additionalBslCredentials == "" {
+				Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
+			}
+
+			// Create Secret for additional BSL
+			secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen)
+			secretKey := fmt.Sprintf("creds-%s", additionalBslProvider)
+			files := map[string]string{
+				secretKey: additionalBslCredentials,
+			}
+
+			Expect(CreateSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed())
+
+			// Create additional BSL using credential
+			additionalBsl := fmt.Sprintf("bsl-%s", uuidgen)
+			Expect(VeleroCreateBackupLocation(context.TODO(),
+				veleroCLI,
+				veleroNamespace,
+				additionalBsl,
+				additionalBslProvider,
+				additionalBslBucket,
+				additionalBslPrefix,
+				additionalBslConfig,
+				secretName,
+				secretKey,
+			)).To(Succeed())
+
+			bsls := []string{"default", additionalBsl}
+
+			for _, bsl := range bsls {
+				backupName = fmt.Sprintf("backup-%s-%s", bsl, uuidgen)
+				restoreName = fmt.Sprintf("restore-%s-%s", bsl, uuidgen)
+
+				Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl)).To(Succeed(),
+					"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
+			}
+		})
 	})
 })

test/e2e/common.go

@@ -2,9 +2,11 @@ package e2e
 
 import (
 	"fmt"
+	"io/ioutil"
 	"os/exec"
 	"time"
 
+	"github.com/pkg/errors"
 	"golang.org/x/net/context"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -44,3 +46,20 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete
 	})
 	return err
 }
+
+func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error {
+	data := make(map[string][]byte)
+
+	for key, filePath := range files {
+		contents, err := ioutil.ReadFile(filePath)
+		if err != nil {
+			return errors.WithMessagef(err, "Failed to read secret file %q", filePath)
+		}
+
+		data[key] = contents
+	}
+
+	secret := builder.ForSecret(namespace, name).Data(data).Result()
+	_, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
+	return err
+}

test/e2e/e2e_suite_test.go

@@ -10,6 +10,7 @@ import (
 
 var (
 	veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace string
+	additionalBslProvider, additionalBslBucket, additionalBslPrefix, additionalBslConfig, additionalBslCredentials                                string
 	installVelero, useVolumeSnapshots                                                                                                             bool
 )
 
@@ -26,6 +27,13 @@ func init() {
 	flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into")
 	flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test.  Optional.")
 	flag.BoolVar(&useVolumeSnapshots, "use-volume-snapshots", false, "Use volume-snapshotter plugin for volume backup.  Optional")
+
+	// Flags to create an additional BSL for multiple credentials test
+	flag.StringVar(&additionalBslProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
+	flag.StringVar(&additionalBslBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
+	flag.StringVar(&additionalBslPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
+	flag.StringVar(&additionalBslConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
+	flag.StringVar(&additionalBslCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
 }
 
 func TestE2e(t *testing.T) {

test/e2e/enable_api_group_versions_test.go

@@ -230,7 +230,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
 		backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
 		namespacesStr := strings.Join(tc.namespaces, ",")
 
-		err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr)
+		err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "")
 		if err != nil {
 			VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup)
 			return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr)

test/e2e/kibishii_tests.go

@@ -85,7 +85,7 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
 }
 
 // RunKibishiiTests runs kibishii tests on the provider.
-func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error {
+func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string) error {
 	fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
 	oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
 	timeout := 10 * time.Minute
@@ -103,7 +103,7 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel
 		return errors.Wrap(err, "Failed to generate data")
 	}
 
-	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace); err != nil {
+	if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation); err != nil {
 		VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName)
 		return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
 	}

test/e2e/velero_utils.go

@@ -210,11 +210,20 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
 }
 
 // VeleroBackupNamespace uses the veleroCLI to backup a namespace.
-func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string) error {
-	backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName,
+func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string) error {
+	args := []string{
+		"--namespace", veleroNamespace,
+		"create", "backup", backupName,
 		"--include-namespaces", namespace,
-		"--default-volumes-to-restic", "--wait")
+		"--default-volumes-to-restic",
+		"--wait",
+	}
+
+	if backupLocation != "" {
+		args = append(args, "--storage-location", backupLocation)
+	}
 
+	backupCmd := exec.CommandContext(ctx, veleroCLI, args...)
 	backupCmd.Stdout = os.Stdout
 	backupCmd.Stderr = os.Stderr
 	fmt.Printf("backup cmd =%v\n", backupCmd)
@@ -314,3 +323,40 @@ func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace st
 	}
 	return nil
 }
+
+func VeleroCreateBackupLocation(ctx context.Context,
+	veleroCLI string,
+	veleroNamespace string,
+	name string,
+	objectStoreProvider string,
+	bucket string,
+	prefix string,
+	config string,
+	secretName string,
+	secretKey string,
+) error {
+	args := []string{
+		"--namespace", veleroNamespace,
+		"create", "backup-location", name,
+		"--provider", objectStoreProvider,
+		"--bucket", bucket,
+	}
+
+	if prefix != "" {
+		args = append(args, "--prefix", prefix)
+	}
+
+	if config != "" {
+		args = append(args, "--config", config)
+	}
+
+	if secretName != "" && secretKey != "" {
+		args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey))
+	}
+
+	bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...)
+	bslCreateCmd.Stdout = os.Stdout
+	bslCreateCmd.Stderr = os.Stderr
+
+	return bslCreateCmd.Run()
+}