Merge pull request #5355 from danfengliu/add-schedule-backup-timing-e…

…2e-test

 Add schedule backup timing E2E test

changelogs/unreleased/5355-danfengliu

@@ -0,0 +1 @@
+Add E2E test for schedule backup

test/e2e/backups/schedule.go

@@ -0,0 +1,166 @@
+package backups
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	. "github.com/vmware-tanzu/velero/test/e2e"
+	. "github.com/vmware-tanzu/velero/test/e2e/test"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/k8s"
+	. "github.com/vmware-tanzu/velero/test/e2e/util/velero"
+)
+
+type ScheduleBackup struct {
+	TestCase
+	ScheduleName   string
+	ScheduleArgs   []string
+	Period         int //Limitation: The unit is minitue only and 60 is divisible by it
+	randBackupName string
+	verifyTimes    int
+}
+
+var ScheduleBackupTest func() = TestFunc(&ScheduleBackup{TestCase: TestCase{NSBaseName: "ns", NSIncluded: &[]string{"ns1"}}})
+
+func (n *ScheduleBackup) Init() error {
+	n.Client = TestClientInstance
+	n.Period = 3
+	n.verifyTimes = 5 // More verify times more confidence
+	n.TestMsg = &TestMSG{
+		Desc:      "Set up a scheduled backup defined by a Cron expression",
+		FailedMSG: "Failed to schedule a backup",
+		Text:      "should backup periodly according to the schedule",
+	}
+	return nil
+}
+
+func (n *ScheduleBackup) StartRun() error {
+
+	n.ScheduleName = n.ScheduleName + "schedule-" + UUIDgen.String()
+	n.RestoreName = n.RestoreName + "restore-ns-mapping-" + UUIDgen.String()
+
+	n.ScheduleArgs = []string{
+		"schedule", "create", "--namespace", VeleroCfg.VeleroNamespace, n.ScheduleName,
+		"--include-namespaces", strings.Join(*n.NSIncluded, ","),
+		"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
+	}
+
+	return nil
+}
+func (n *ScheduleBackup) CreateResources() error {
+	n.Ctx, _ = context.WithTimeout(context.Background(), 60*time.Minute)
+	for _, ns := range *n.NSIncluded {
+		By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() {
+			Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns))
+		})
+		configmaptName := n.NSBaseName
+		fmt.Printf("Creating configmap %s in namespaces ...%s\n", configmaptName, ns)
+		_, err := CreateConfigMap(n.Client.ClientGo, ns, configmaptName, nil)
+		Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", ns))
+		Expect(WaitForConfigMapComplete(n.Client.ClientGo, ns, configmaptName)).To(Succeed(),
+			fmt.Sprintf("ailed to ensure secret completion in namespace: %q", ns))
+	}
+	return nil
+}
+
+func (n *ScheduleBackup) Backup() error {
+	// Wait until the beginning of the given period to create schedule, it will give us
+	//   a predictable period to wait for the first scheduled backup, and verify no immediate
+	//   scheduled backup was created between schedule creation and first scheduled backup.
+	By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() {
+		for i := 0; i < n.Period*60/30; i++ {
+			time.Sleep(30 * time.Second)
+			now := time.Now().Minute()
+			triggerNow := now % n.Period
+			if triggerNow == 0 {
+				Expect(VeleroCmdExec(n.Ctx, VeleroCfg.VeleroCLI, n.ScheduleArgs)).To(Succeed())
+				break
+			}
+		}
+	})
+	return nil
+}
+func (n *ScheduleBackup) Destroy() error {
+	By(fmt.Sprintf("Schedule %s is created without any delay\n", n.ScheduleName), func() {
+		creationTimestamp, err := GetSchedule(context.Background(), VeleroCfg.VeleroNamespace, n.ScheduleName)
+		Expect(err).To(Succeed())
+
+		creationTime, err := time.Parse(time.RFC3339, strings.Replace(creationTimestamp, "'", "", -1))
+		Expect(err).To(Succeed())
+		fmt.Printf("Schedule %s created at %s\n", n.ScheduleName, creationTime)
+		now := time.Now()
+		diff := creationTime.Sub(now)
+		Expect(diff.Minutes() < 1).To(Equal(true))
+	})
+
+	By(fmt.Sprintf("No immediate backup is created by schedule %s\n", n.ScheduleName), func() {
+		for i := 0; i < n.Period; i++ {
+			time.Sleep(1 * time.Minute)
+			now := time.Now()
+			fmt.Printf("Get backup for #%d time at %v\n", i, now)
+			//Ignore the last minute in the period avoiding met the 1st backup by schedule
+			if i != n.Period-1 {
+				backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
+				Expect(err).To(Succeed())
+				Expect(len(backupsInfo) == 0).To(Equal(true))
+			}
+		}
+	})
+
+	By("Delay one more minute to make sure the new backup was created in the given period", func() {
+		time.Sleep(1 * time.Minute)
+	})
+
+	By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() {
+		for i := 0; i < n.verifyTimes; i++ {
+			fmt.Printf("Start to sleep %d minute #%d time...\n", n.Period, i+1)
+			time.Sleep(time.Duration(n.Period) * time.Minute)
+			bMap := make(map[string]string)
+			backupsInfo, err := GetScheduledBackupsCreationTime(context.Background(), VeleroCfg.VeleroCLI, "default", n.ScheduleName)
+			Expect(err).To(Succeed())
+			Expect(len(backupsInfo) == i+2).To(Equal(true))
+			for index, bi := range backupsInfo {
+				bList := strings.Split(bi, ",")
+				fmt.Printf("Backup %d: %v\n", index, bList)
+				bMap[bList[0]] = bList[1]
+				_, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1])
+				Expect(err).To(Succeed())
+			}
+			if i == n.verifyTimes-1 {
+				backupInfo := backupsInfo[rand.Intn(len(backupsInfo))]
+				n.randBackupName = strings.Split(backupInfo, ",")[0]
+			}
+		}
+	})
+
+	n.BackupName = strings.Replace(n.randBackupName, " ", "", -1)
+
+	By("Delete all namespaces", func() {
+		Expect(CleanupNamespacesWithPoll(n.Ctx, n.Client, n.NSBaseName)).To(Succeed(), "Could cleanup retrieve namespaces")
+	})
+
+	n.RestoreArgs = []string{
+		"create", "--namespace", VeleroCfg.VeleroNamespace, "restore", n.RestoreName,
+		"--from-backup", n.BackupName,
+		"--wait",
+	}
+
+	return nil
+}
+
+func (n *ScheduleBackup) Verify() error {
+	By("Namespaces were restored", func() {
+		for _, ns := range *n.NSIncluded {
+			configmap, err := GetConfigmap(n.Client.ClientGo, ns, n.NSBaseName)
+			fmt.Printf("Restored configmap is %v\n", configmap)
+			Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("failed to list configmap in namespace: %q\n", ns))
+		}
+
+	})
+	return nil
+}

test/e2e/e2e_suite_test.go

@@ -111,6 +111,7 @@ var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup dele
 var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots)
 var _ = Describe("[Backups][TTL] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest)
 var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest)
+var _ = Describe("[Backups][Schedule] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest)
 
 var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest)
 

test/e2e/util/common/common.go

@@ -38,7 +38,7 @@ func GetListBy2Pipes(ctx context.Context, cmdline1, cmdline2, cmdline3 OsCommand
 	_ = c2.Wait()
 	_ = c3.Wait()
 
-	fmt.Println(&b2)
+	//fmt.Println(&b2)
 	scanner := bufio.NewScanner(&b2)
 	var ret []string
 	for scanner.Scan() {

test/e2e/util/k8s/common.go

@@ -63,7 +63,7 @@ func WaitForPods(ctx context.Context, client TestClient, namespace string, pods
 			checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(context.TODO(), podName, metav1.GetOptions{})
 			if err != nil {
 				//Should ignore "etcdserver: request timed out" kind of errors, try to get pod status again before timeout.
-				fmt.Println(errors.Wrap(err, fmt.Sprintf("Failed to verify pod %s/%s is %s, try again...", namespace, podName, corev1api.PodRunning)))
+				fmt.Println(errors.Wrap(err, fmt.Sprintf("Failed to verify pod %s/%s is %s, try again...\n", namespace, podName, corev1api.PodRunning)))
 				return false, nil
 			}
 			// If any pod is still waiting we don't need to check any more so return and wait for next poll interval

test/e2e/util/k8s/namespace.go

@@ -100,7 +100,7 @@ func CleanupNamespacesWithPoll(ctx context.Context, client TestClient, nsBaseNam
 			if err != nil {
 				return errors.Wrapf(err, "Could not delete namespace %s", checkNamespace.Name)
 			}
-			fmt.Printf("Delete namespace %s", checkNamespace.Name)
+			fmt.Printf("Delete namespace %s\n", checkNamespace.Name)
 		}
 	}
 	return nil

test/e2e/util/velero/velero_utils.go

@@ -873,6 +873,43 @@ func GetBackupsFromBsl(ctx context.Context, veleroCLI, bslName string) ([]string
 	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
 }
 
+func GetScheduledBackupsCreationTime(ctx context.Context, veleroCLI, bslName, scheduleName string) ([]string, error) {
+	var creationTimes []string
+	backups, err := GetBackupsCreationTime(ctx, veleroCLI, bslName)
+	if err != nil {
+		return nil, err
+	}
+	for _, b := range backups {
+		if strings.Contains(b, scheduleName) {
+			creationTimes = append(creationTimes, b)
+		}
+	}
+	return creationTimes, nil
+}
+func GetBackupsCreationTime(ctx context.Context, veleroCLI, bslName string) ([]string, error) {
+	args1 := []string{"get", "backups"}
+	createdTime := "$1,\",\" $5,$6,$7,$8"
+	if strings.TrimSpace(bslName) != "" {
+		args1 = append(args1, "-l", "velero.io/storage-location="+bslName)
+	}
+	CmdLine1 := &common.OsCommandLine{
+		Cmd:  veleroCLI,
+		Args: args1,
+	}
+
+	CmdLine2 := &common.OsCommandLine{
+		Cmd:  "awk",
+		Args: []string{"{print " + createdTime + "}"},
+	}
+
+	CmdLine3 := &common.OsCommandLine{
+		Cmd:  "tail",
+		Args: []string{"-n", "+2"},
+	}
+
+	return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3)
+}
+
 func GetAllBackups(ctx context.Context, veleroCLI string) ([]string, error) {
 	return GetBackupsFromBsl(ctx, veleroCLI, "")
 }
@@ -978,23 +1015,15 @@ func GetSnapshotCheckPoint(client TestClient, VeleroCfg VerleroConfig, expectCou
 }
 
 func GetBackupTTL(ctx context.Context, veleroNamespace, backupName string) (string, error) {
-
 	checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
 		"get", "backup", "-n", veleroNamespace, backupName, "-o=jsonpath='{.spec.ttl}'")
 	fmt.Printf("checkSnapshotCmd cmd =%v\n", checkSnapshotCmd)
 	stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd)
 	if err != nil {
 		fmt.Print(stdout)
 		fmt.Print(stderr)
-		return "", errors.Wrap(err, "failed to verify")
+		return "", errors.Wrap(err, fmt.Sprintf("failed to run command %s", checkSnapshotCmd))
 	}
-	// lines := strings.Split(stdout, "\n")
-	// complete := true
-	// for _, curLine := range lines {
-	// 	fmt.Println(curLine)
-
-	// }
-	// return complete, nil
 	return stdout, err
 }
 
@@ -1021,10 +1050,23 @@ func DeleteBackups(ctx context.Context, client TestClient) error {
 		return fmt.Errorf("failed to list backup object in %s namespace with err %v", VeleroCfg.VeleroNamespace, err)
 	}
 	for _, backup := range backupList.Items {
-		fmt.Printf("Backup %s is going to be deleted...", backup.Name)
+		fmt.Printf("Backup %s is going to be deleted...\n", backup.Name)
 		if err := VeleroBackupDelete(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, backup.Name); err != nil {
 			return err
 		}
 	}
 	return nil
 }
+
+func GetSchedule(ctx context.Context, veleroNamespace, scheduleName string) (string, error) {
+	checkSnapshotCmd := exec.CommandContext(ctx, "kubectl",
+		"get", "schedule", "-n", veleroNamespace, scheduleName, "-o=jsonpath='{.metadata.creationTimestamp}'")
+	fmt.Printf("Cmd =%v\n", checkSnapshotCmd)
+	stdout, stderr, err := veleroexec.RunCommand(checkSnapshotCmd)
+	if err != nil {
+		fmt.Print(stdout)
+		fmt.Print(stderr)
+		return "", errors.Wrap(err, fmt.Sprintf("failed to run command %s", checkSnapshotCmd))
+	}
+	return stdout, err
+}