Skip to content

Commit

Permalink
Add E2E test for multiple credentials
Browse files Browse the repository at this point in the history
This change adds an E2E test which exercises the mulitple credentials
feature added in #3489. The test creates a secret from the given
credentials and creates a BackupStorageLocation which uses those
credentials. A backup and restore is then performed to the default
BSL and to the newly created BSL.

This change adds new flags to the E2E test suite to configure the BSL
created and used in the test.

Signed-off-by: Bridget McErlean <[email protected]>
  • Loading branch information
zubron committed Mar 10, 2021
1 parent 55a9b65 commit 3b969b0
Show file tree
Hide file tree
Showing 10 changed files with 165 additions and 13 deletions.
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ require (
github.com/hashicorp/go-plugin v0.0.0-20190610192547-a1bc61569a26
github.com/joho/godotenv v1.3.0
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0
github.com/onsi/ginkgo v1.15.0
github.com/onsi/ginkgo v1.15.1
github.com/onsi/gomega v1.10.2
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.7.1
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -419,6 +419,8 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h
github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
Expand All @@ -429,8 +431,8 @@ github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.1/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4=
github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg=
github.com/onsi/ginkgo v1.15.1 h1:DsXNrKujDlkMS9Rsxmd+Fg7S6Kc5lhE+qX8tY6laOxc=
github.com/onsi/ginkgo v1.15.1/go.mod h1:Dd6YFfwBW84ETqqtL0CPyPXillHgY6XhQH3uuCCTr/o=
github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
Expand Down
14 changes: 13 additions & 1 deletion test/e2e/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,13 @@ VSL_CONFIG ?=
CLOUD_PROVIDER ?=
OBJECT_STORE_PROVIDER ?=

# Flags to create an additional BSL for multiple credentials tests
ADDITIONAL_OBJECT_STORE_PROVIDER ?=
ADDITIONAL_CREDS_FILE ?=
ADDITIONAL_BSL_BUCKET ?=
ADDITIONAL_BSL_PREFIX ?=
ADDITIONAL_BSL_CONFIG ?=

.PHONY:ginkgo
ginkgo: # Make sure ginkgo is in $GOPATH/bin
go get github.com/onsi/ginkgo/ginkgo
Expand All @@ -78,7 +85,12 @@ run: ginkgo
-bsl-config=$(BSL_CONFIG) \
-vsl-config=$(VSL_CONFIG) \
-cloud-provider=$(CLOUD_PROVIDER) \
-object-store-provider="$(OBJECT_STORE_PROVIDER)"
-object-store-provider="$(OBJECT_STORE_PROVIDER)" \
-additional-bsl-object-store-provider="$(ADDITIONAL_OBJECT_STORE_PROVIDER)" \
-additional-bsl-credentials-file=$(ADDITIONAL_CREDS_FILE) \
-additional-bsl-bucket=$(ADDITIONAL_BSL_BUCKET) \
-additional-bsl-prefix=$(ADDITIONAL_BSL_PREFIX) \
-additional-bsl-config=$(ADDITIONAL_BSL_CONFIG)

build: ginkgo
mkdir -p $(OUTPUT_DIR)
Expand Down
17 changes: 17 additions & 0 deletions test/e2e/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ the object-store-provider to be specified.
1. `-vsl-config`: Configuration to use for the volume snapshot location. Format is key1=value1,key2=value2. Optional.
1. `-velero-namespace`: Namespace to install velero in. Optional, defaults to "velero".
1. `-install-velero`: Specifies whether to install/uninstall velero for the tests. Optional, defaults to "true".
1. `-additional-bsl-object-store-provider`: Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.
1. `-additional-bsl-bucket`: Name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.
1. `-additional-bsl-prefix`: Prefix in the `additional-bsl-bucket`, under which all Velero data should be stored. Optional.
1. `-additional-bsl-config`: Configuration to use for the additional backup storage location. Format is key1=value1,key2=value2. Optional.
1. `-additional-bsl-credentials-file`: File containing credentials for the additional backup storage location. Required if testing multiple credentials support.

These configurations or parameters are used to generate install options for Velero for each test suite.

Expand All @@ -62,6 +67,11 @@ Below is a mapping between `make` variables to E2E configuration flags.
1. `BSL_PREFIX`: `-prefix`. Optional.
1. `BSL_CONFIG`: `-bsl-config`. Optional.
1. `VSL_CONFIG`: `-vsl-config`. Optional.
1. `ADDITIONAL_OBJECT_STORE_PROVIDER`: `-additional-bsl-object-store-provider`. Optional.
1. `ADDITIONAL_CREDS_FILE`: `-additional-bsl-bucket`. Optional.
1. `ADDITIONAL_BSL_BUCKET`: `-additional-bsl-prefix`. Optional.
1. `ADDITIONAL_BSL_PREFIX`: `-additional-bsl-config`. Optional.
1. `ADDITIONAL_BSL_CONFIG`: `-additional-bsl-credentials-file`. Optional.

For example, E2E tests can be run from Velero repository roots using the commands below:

Expand All @@ -82,6 +92,13 @@ For example, E2E tests can be run from Velero repository roots using the command
```bash
BSL_CONFIG="region=minio,s3ForcePathStyle=\"true\",s3Url=http://192.168.1.124:9000" BSL_PREFIX=veldat BSL_BUCKET=velero CREDS_FILE=~/go/src/github.com/vmware-tanzu/velero/frankie-secrets/credentials-minio PLUGIN_PROVIDER=aws VELERO_IMAGE=projects.registry.vmware.com/tanzu_migrator/velero-pr3133:0.0.5 GINKGO_FOCUS="API group versions" make test-e2e
```
1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider and use Microsoft Azure as the storage provider for an additional Backup Storage Location:
```bash
make test-e2e \
CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws BSL_BUCKET=<BUCKET_FOR_E2E_TEST_BACKUP> BSL_PREFIX=<PREFIX_UNDER_BUCKET> CREDS_FILE=/path/to/aws-creds \
ADDITIONAL_OBJECT_STORE_PROVIDER=azure ADDITIONAL_BSL_BUCKET=<BUCKET_FOR_AZURE_BSL> ADDITIONAL_BSL_PREFIX=<PREFIX_UNDER_BUCKET> ADDITIONAL_BSL_CONFIG=<CONFIG_FOR_AZURE_BUCKET> ADDITIONAL_CREDS_FILE=/path/to/azure-creds
```
Please refer to `velero-plugin-for-microsoft-azure` documentation for instruction to [set up permissions for Velero](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#set-permissions-for-velero) and to [set up azure storage account and blob container](https://github.com/vmware-tanzu/velero-plugin-for-microsoft-azure#setup-azure-storage-account-and-blob-container)

## Filtering tests

Expand Down
54 changes: 51 additions & 3 deletions test/e2e/backup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package e2e
import (
"context"
"flag"
"fmt"
"time"

"github.com/google/uuid"
Expand Down Expand Up @@ -49,14 +50,61 @@ var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for

})

Context("When kibishii is the sample workload", func() {
It("should be successfully backed up and restored", func() {
When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
backupName = "backup-" + uuidgen.String()
restoreName = "restore-" + uuidgen.String()
// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
// KinD. So use the kind installation for Kibishii.
Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName)).To(Succeed(),
Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "")).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})

It("should successfully back up and restore to multiple BackupStorageLocations with unique credentials", func() {
if additionalBslProvider == "" {
Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
}

if additionalBslBucket == "" {
Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
}

if additionalBslCredentials == "" {
Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
}

// Create Secret for additional BSL
secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen)
secretKey := fmt.Sprintf("creds-%s", additionalBslProvider)
files := map[string]string{
secretKey: additionalBslCredentials,
}

Expect(CreateSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed())

// Create additional BSL using credential
additionalBsl := fmt.Sprintf("bsl-%s", uuidgen)
Expect(VeleroCreateBackupLocation(context.TODO(),
veleroCLI,
veleroNamespace,
additionalBsl,
additionalBslProvider,
additionalBslBucket,
additionalBslPrefix,
additionalBslConfig,
secretName,
secretKey,
)).To(Succeed())

bsls := []string{"default", additionalBsl}

for _, bsl := range bsls {
backupName = fmt.Sprintf("backup-%s-%s", bsl, uuidgen)
restoreName = fmt.Sprintf("restore-%s-%s", bsl, uuidgen)

Expect(RunKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace to BSL %s", bsl)
}
})
})
})
19 changes: 19 additions & 0 deletions test/e2e/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import (
"os/exec"
"time"

"github.com/pkg/errors"
"golang.org/x/net/context"
"io/ioutil"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -44,3 +46,20 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete
})
return err
}

func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error {
data := make(map[string][]byte)

for key, filePath := range files {
contents, err := ioutil.ReadFile(filePath)
if err != nil {
return errors.WithMessagef(err, "Failed to read secret file %q", filePath)
}

data[key] = contents
}

secret := builder.ForSecret(namespace, name).Data(data).Result()
_, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{})
return err
}
8 changes: 8 additions & 0 deletions test/e2e/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

var (
veleroCLI, veleroImage, cloudCredentialsFile, bslConfig, bslBucket, bslPrefix, vslConfig, cloudProvider, objectStoreProvider, veleroNamespace string
additionalBslProvider, additionalBslBucket, additionalBslPrefix, additionalBslConfig, additionalBslCredentials string
installVelero, useVolumeSnapshots bool
)

Expand All @@ -26,6 +27,13 @@ func init() {
flag.StringVar(&veleroNamespace, "velero-namespace", "velero", "Namespace to install Velero into")
flag.BoolVar(&installVelero, "install-velero", true, "Install/uninstall velero during the test. Optional.")
flag.BoolVar(&useVolumeSnapshots, "use-volume-snapshots", false, "Use volume-snapshotter plugin for volume backup. Optional")

// Flags to create an additional BSL for multiple credentials test
flag.StringVar(&additionalBslProvider, "additional-bsl-object-store-provider", "", "Provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&additionalBslBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.")
flag.StringVar(&additionalBslPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.")
flag.StringVar(&additionalBslConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
flag.StringVar(&additionalBslCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
}

func TestE2e(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/enable_api_group_versions_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -230,7 +230,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string,
backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
namespacesStr := strings.Join(tc.namespaces, ",")

err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr)
err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "")
if err != nil {
VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup)
return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr)
Expand Down
4 changes: 2 additions & 2 deletions test/e2e/kibishii_tests.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel
}

// RunKibishiiTests runs kibishii tests on the provider.
func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName string) error {
func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string) error {
fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute)
oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60)
timeout := 10 * time.Minute
Expand All @@ -103,7 +103,7 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel
return errors.Wrap(err, "Failed to generate data")
}

if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace); err != nil {
if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation); err != nil {
VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName)
return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace)
}
Expand Down
52 changes: 49 additions & 3 deletions test/e2e/velero_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,11 +210,20 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st
}

// VeleroBackupNamespace uses the veleroCLI to backup a namespace.
func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string) error {
backupCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "backup", backupName,
func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string) error {
args := []string{
"--namespace", veleroNamespace,
"create", "backup", backupName,
"--include-namespaces", namespace,
"--default-volumes-to-restic", "--wait")
"--default-volumes-to-restic",
"--wait",
}

if backupLocation != "" {
args = append(args, "--storage-location", backupLocation)
}

backupCmd := exec.CommandContext(ctx, veleroCLI, args...)
backupCmd.Stdout = os.Stdout
backupCmd.Stderr = os.Stderr
fmt.Printf("backup cmd =%v\n", backupCmd)
Expand Down Expand Up @@ -314,3 +323,40 @@ func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace st
}
return nil
}

func VeleroCreateBackupLocation(ctx context.Context,
veleroCLI string,
veleroNamespace string,
name string,
objectStoreProvider string,
bucket string,
prefix string,
config string,
secretName string,
secretKey string,
) error {
args := []string{
"--namespace", veleroNamespace,
"create", "backup-location", name,
"--provider", objectStoreProvider,
"--bucket", bucket,
}

if prefix != "" {
args = append(args, "--prefix", prefix)
}

if config != "" {
args = append(args, "--config", config)
}

if secretName != "" && secretKey != "" {
args = append(args, "--credential", fmt.Sprintf("%s=%s", secretName, secretKey))
}

bslCreateCmd := exec.CommandContext(ctx, veleroCLI, args...)
bslCreateCmd.Stdout = os.Stdout
bslCreateCmd.Stderr = os.Stderr

return bslCreateCmd.Run()
}

0 comments on commit 3b969b0

Please sign in to comment.