Skip to content

Commit

Permalink
Adjust structure of e2e test codes
Browse files Browse the repository at this point in the history 
Put every test moduels into seperate packages and all velero install parameters integrated into one struct

Signed-off-by: Ming <[email protected]>
  • Loading branch information
@danfengliu
ming qiu authored and danfengliu committed Jan 24, 2022
1 parent 8f1bfc3 commit c9de6b1
Show file tree
Hide file tree
Showing 15 changed files with 566 additions and 496 deletions.
1 change: 1 addition & 0 deletions changelogs/unreleased/4386-mqiu
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Adjust structure of e2e test codes
84 changes: 39 additions & 45 deletions test/e2e/backup_test.go → test/e2e/backup/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package e2e
package backup

import (
"context"
Expand All @@ -23,98 +23,93 @@ import (
"github.com/google/uuid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)

var (
uuidgen uuid.UUID
. "github.com/vmware-tanzu/velero/test/e2e"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
kibishiiutils "github.com/vmware-tanzu/velero/test/e2e/util/kibishii"
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)

// Test backup and restore of Kibishi using restic
var _ = Describe("[Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", backup_restore_with_restic)

var _ = Describe("[Snapshot] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", backup_restore_with_snapshots)

func backup_restore_with_snapshots() {
backup_restore_test(true)
func BackupRestoreWithSnapshots() {
BackupRestoreTest(true)
}

func backup_restore_with_restic() {
backup_restore_test(false)
func BackupRestoreWithRestic() {
BackupRestoreTest(false)
}

func backup_restore_test(useVolumeSnapshots bool) {
func BackupRestoreTest(useVolumeSnapshots bool) {
var (
backupName, restoreName string
)

client, err := newTestClient()
client, err := k8sutils.NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests")

BeforeEach(func() {
if useVolumeSnapshots && cloudProvider == "kind" {
if useVolumeSnapshots && VeleroCfg.CloudProvider == "kind" {
Skip("Volume snapshots not supported on kind")
}
var err error
flag.Parse()
uuidgen, err = uuid.NewRandom()
UUIDgen, err = uuid.NewRandom()
Expect(err).To(Succeed())
if installVelero {
Expect(veleroInstall(context.Background(), veleroCLI, veleroImage, resticHelperImage, plugins, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots,
cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, crdsVersion, "", registryCredentialFile)).To(Succeed())
if VeleroCfg.InstallVelero {
Expect(veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "", useVolumeSnapshots)).To(Succeed())
}
})

AfterEach(func() {
if installVelero {
err = veleroUninstall(context.Background(), veleroCLI, veleroNamespace)
if VeleroCfg.InstallVelero {
err = veleroutils.VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).To(Succeed())
}
})

When("kibishii is the sample workload", func() {
It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
backupName = "backup-" + uuidgen.String()
restoreName = "restore-" + uuidgen.String()
backupName = "backup-" + UUIDgen.String()
restoreName = "restore-" + UUIDgen.String()
// Even though we are using Velero's CloudProvider plugin for object storage, the kubernetes cluster is running on
// KinD. So use the kind installation for Kibishii.
Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, "", useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, "", useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace")
})

It("should successfully back up and restore to an additional BackupStorageLocation with unique credentials", func() {
if additionalBSLProvider == "" {
if VeleroCfg.AdditionalBSLProvider == "" {
Skip("no additional BSL provider given, not running multiple BackupStorageLocation with unique credentials tests")
}

if additionalBSLBucket == "" {
if VeleroCfg.AdditionalBSLBucket == "" {
Skip("no additional BSL bucket given, not running multiple BackupStorageLocation with unique credentials tests")
}

if additionalBSLCredentials == "" {
if VeleroCfg.AdditionalBSLCredentials == "" {
Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests")
}

Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider, addBSLPlugins)).To(Succeed())
Expect(veleroutils.VeleroAddPluginsForProvider(context.TODO(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, VeleroCfg.AdditionalBSLProvider, VeleroCfg.AddBSLPlugins)).To(Succeed())

// Create Secret for additional BSL
secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen)
secretKey := fmt.Sprintf("creds-%s", additionalBSLProvider)
secretName := fmt.Sprintf("bsl-credentials-%s", UUIDgen)
secretKey := fmt.Sprintf("creds-%s", VeleroCfg.AdditionalBSLProvider)
files := map[string]string{
secretKey: additionalBSLCredentials,
secretKey: VeleroCfg.AdditionalBSLCredentials,
}

Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed())
Expect(k8sutils.CreateSecretFromFiles(context.TODO(), client, VeleroCfg.VeleroNamespace, secretName, files)).To(Succeed())

// Create additional BSL using credential
additionalBsl := fmt.Sprintf("bsl-%s", uuidgen)
Expect(veleroCreateBackupLocation(context.TODO(),
veleroCLI,
veleroNamespace,
additionalBsl := fmt.Sprintf("bsl-%s", UUIDgen)
Expect(veleroutils.VeleroCreateBackupLocation(context.TODO(),
VeleroCfg.VeleroCLI,
VeleroCfg.VeleroNamespace,
additionalBsl,
additionalBSLProvider,
additionalBSLBucket,
additionalBSLPrefix,
additionalBSLConfig,
VeleroCfg.AdditionalBSLProvider,
VeleroCfg.AdditionalBSLBucket,
VeleroCfg.AdditionalBSLPrefix,
VeleroCfg.AdditionalBSLConfig,
secretName,
secretKey,
)).To(Succeed())
Expand All @@ -127,11 +122,10 @@ func backup_restore_test(useVolumeSnapshots bool) {
// We limit the length of backup name here to avoid the issue of vsphere plugin https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/370
// We can remove the logic once the issue is fixed
if bsl == "default" {
backupName = fmt.Sprintf("%s-%s", backupName, uuidgen)
restoreName = fmt.Sprintf("%s-%s", restoreName, uuidgen)
backupName = fmt.Sprintf("%s-%s", backupName, UUIDgen)
restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen)
}

Expect(runKibishiiTests(client, cloudProvider, veleroCLI, veleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, registryCredentialFile)).To(Succeed(),
Expect(kibishiiutils.RunKibishiiTests(client, VeleroCfg.CloudProvider, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backupName, restoreName, bsl, useVolumeSnapshots, VeleroCfg.RegistryCredentialFile)).To(Succeed(),
"Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl)
}
})
Expand Down
72 changes: 29 additions & 43 deletions test/e2e/enable_api_group_versions_test.go → test/e2e/basic/enable_api_group_versions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,12 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e
package basic

import (
"context"
"encoding/json"
"flag"
"fmt"
"os/exec"
"strconv"
Expand All @@ -34,46 +35,31 @@ import (

"github.com/vmware-tanzu/velero/pkg/builder"
veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec"
. "github.com/vmware-tanzu/velero/test/e2e"
k8sutils "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
veleroutils "github.com/vmware-tanzu/velero/test/e2e/util/velero"
)

var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() {
func APIGropuVersionsTest() {
var (
resource, group string
err error
ctx = context.Background()
)

client, err := newTestClient()
client, err := k8sutils.NewTestClient()
Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests")

BeforeEach(func() {
resource = "rockbands"
group = "music.example.io"

uuidgen, err = uuid.NewRandom()
UUIDgen, err = uuid.NewRandom()
Expect(err).NotTo(HaveOccurred())

flag.Parse()
// TODO: install Velero once for the test suite once feature flag is
// removed and velero installation becomes the same as other e2e tests.
if installVelero {
err = veleroInstall(
context.Background(),
veleroCLI,
veleroImage,
resticHelperImage,
plugins,
veleroNamespace,
cloudProvider,
objectStoreProvider,
false,
cloudCredentialsFile,
bslBucket,
bslPrefix,
bslConfig,
vslConfig,
crdsVersion,
"EnableAPIGroupVersions", // TODO: remove when feature flag is removed
registryCredentialFile)
if VeleroCfg.InstallVelero {
err = veleroutils.VeleroInstall(context.Background(), &VeleroCfg, "EnableAPIGroupVersions", false)
Expect(err).NotTo(HaveOccurred())
}
})
Expand All @@ -88,8 +74,8 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
}
Expect(err).NotTo(HaveOccurred())

if installVelero {
err = veleroUninstall(ctx, veleroCLI, veleroNamespace)
if VeleroCfg.InstallVelero {
err = veleroutils.VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)
Expect(err).NotTo(HaveOccurred())
}

Expand All @@ -105,9 +91,9 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions",
)).To(Succeed(), "Failed to successfully backup and restore multiple API Groups")
})
})
})
}

func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string) error {
func runEnableAPIGroupVersionsTests(ctx context.Context, client k8sutils.TestClient, resource, group string) error {
tests := []struct {
name string
namespaces []string
Expand Down Expand Up @@ -180,7 +166,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
},
tgtCrdYaml: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml",
tgtVer: "v2beta1",
cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data(
cm: builder.ForConfigMap(VeleroCfg.VeleroNamespace, "enableapigroupversions").Data(
"restoreResourcesVersionPriority",
`rockbands.music.example.io=v2beta1,v2beta2,v2`,
).Result(),
Expand Down Expand Up @@ -224,11 +210,11 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
for version, cr := range tc.srcCRs {
ns := resource + "-src-" + version

if err := createNamespace(ctx, client, ns); err != nil {
if err := k8sutils.CreateNamespace(ctx, client, ns); err != nil {
return errors.Wrapf(err, "create %s namespace", ns)
}
defer func(namespace string) {
if err = deleteNamespace(ctx, client, namespace, true); err != nil {
if err = k8sutils.DeleteNamespace(ctx, client, namespace, true); err != nil {
fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))
}
}(ns)
Expand All @@ -243,16 +229,16 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
// Restart Velero pods in order to recognize music-system CRD right away
// instead of waiting for discovery helper to refresh. See
// https://github.com/vmware-tanzu/velero/issues/3471.
if err := restartPods(ctx, veleroNamespace); err != nil {
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "restart Velero pods")
}

backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
backup := "backup-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)
namespacesStr := strings.Join(tc.namespaces, ",")

err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false)
err = veleroutils.VeleroBackupNamespace(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, namespacesStr, "", false)
if err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, backup, "")
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup, "")
return errors.Wrapf(err, "back up %s namespaces on source cluster", namespacesStr)
}

Expand All @@ -261,7 +247,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
}

for _, ns := range tc.namespaces {
if err := deleteNamespace(ctx, client, ns, true); err != nil {
if err := k8sutils.DeleteNamespace(ctx, client, ns, true); err != nil {
return errors.Wrapf(err, "delete %s namespace from source cluster", ns)
}
}
Expand All @@ -275,23 +261,23 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso

// Apply config map if there is one.
if tc.cm != nil {
_, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
_, err := client.ClientGo.CoreV1().ConfigMaps(VeleroCfg.VeleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{})
if err != nil {
return errors.Wrap(err, "create config map with user version priorities")
}
}

// Reset Velero to recognize music-system CRD.
if err := restartPods(ctx, veleroNamespace); err != nil {
if err := restartPods(ctx, VeleroCfg.VeleroNamespace); err != nil {
return errors.Wrapf(err, "restart Velero pods")
}

// Restore rockbands namespaces.
restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i)
restore := "restore-rockbands-" + UUIDgen.String() + "-" + strconv.Itoa(i)

if tc.want != nil {
if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil {
runDebug(context.Background(), veleroCLI, veleroNamespace, "", restore)
if err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup); err != nil {
veleroutils.RunDebug(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, "", restore)
return errors.Wrapf(err, "restore %s namespaces on target cluster", namespacesStr)
}

Expand Down Expand Up @@ -329,7 +315,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso
} else {
// No custom resource should have been restored. Expect "no resource found"
// error during restore.
err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup)
err := veleroutils.VeleroRestore(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, restore, backup)

if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" {
return errors.New("expected error but not none")
Expand Down
Loading

0 comments on commit c9de6b1

Please sign in to comment.