From e6c94af3587e466b65410be4cee099d548c0f9b1 Mon Sep 17 00:00:00 2001 From: danfengl Date: Thu, 18 Aug 2022 09:18:24 +0000 Subject: [PATCH] Add enable API group on k8s resources E2E test upon issue #5146 Signed-off-by: danfengl --- test/e2e/basic/enable_api_group_versions.go | 141 +++++++++++++++++- test/e2e/e2e_suite_test.go | 2 +- test/e2e/migration/migration.go | 2 +- .../case-a-source-v1beta1.yaml | 90 +++++++++++ test/e2e/util/k8s/common.go | 65 +++++++- 5 files changed, 295 insertions(+), 5 deletions(-) create mode 100644 test/e2e/testdata/enable_api_group_versions/case-a-source-v1beta1.yaml diff --git a/test/e2e/basic/enable_api_group_versions.go b/test/e2e/basic/enable_api_group_versions.go index 220f6a0298..5c4d11c348 100644 --- a/test/e2e/basic/enable_api_group_versions.go +++ b/test/e2e/basic/enable_api_group_versions.go @@ -40,6 +40,128 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/util/velero" ) +func APIExtensionsVersionsTest() { + var ( + backupName, restoreName string + ) + resourceName := "apiextensions.k8s.io" + crdName := "rocknrollbands.music.example.io" + label := "for=backup" + srcCrdYaml := "testdata/enable_api_group_versions/case-a-source-v1beta1.yaml" + BeforeEach(func() { + if VeleroCfg.DefaultCluster == "" && VeleroCfg.StandbyCluster == "" { + Skip("CRD with apiextension versions migration test needs 2 clusters") + } + Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) + srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName) + Expect(err).ShouldNot(HaveOccurred()) + dstVersions, err := GetAPIVersions(VeleroCfg.StandbyClient, resourceName) + Expect(err).ShouldNot(HaveOccurred()) + + Expect(srcVersions).Should(ContainElement("v1"), func() string { + Skip("CRD with apiextension versions srcVersions should have v1") + return "" + }) + Expect(srcVersions).Should(ContainElement("v1beta1"), func() string { + Skip("CRD with apiextension versions srcVersions should have v1") + return "" + }) + Expect(dstVersions).Should(ContainElement("v1"), func() string { + Skip("CRD with apiextension versions dstVersions should have v1") + return "" + }) + Expect(len(srcVersions) > 1 && len(dstVersions) == 1).Should(Equal(true), func() string { + Skip("Source cluster should support apiextension v1 and v1beta1, destination cluster should only support apiextension v1") + return "" + }) + }) + AfterEach(func() { + if !VeleroCfg.Debug { + By("Clean backups after test", func() { + DeleteBackups(context.Background(), *VeleroCfg.DefaultClient) + }) + if VeleroCfg.InstallVelero { + By("Uninstall Velero and delete CRD ", func() { + Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed()) + + Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed()) + Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(deleteCRDByName(context.Background(), crdName)).To(Succeed()) + }) + } + By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) + VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient + }) + } + + }) + Context("When EnableAPIGroupVersions flag is set", func() { + It("Enable API Group to B/R CRD APIExtensionsVersions", func() { + backupName = "backup-" + UUIDgen.String() + restoreName = "restore-" + UUIDgen.String() + + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", VeleroCfg.DefaultCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) + VeleroCfg.ObjectStoreProvider = "" + VeleroCfg.Features = "EnableAPIGroupVersions" + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + }) + + By(fmt.Sprintf("Install CRD of apiextenstions v1beta1 in cluster-A (%s)", VeleroCfg.DefaultCluster), func() { + Expect(installCRD(context.Background(), srcCrdYaml)).To(Succeed()) + Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed()) + Expect(AddLabelToCRD(context.Background(), crdName, label)).To(Succeed()) + }) + + By("Backup CRD", func() { + var BackupCfg BackupConfig + BackupCfg.BackupName = backupName + BackupCfg.IncludeResources = "crd" + BackupCfg.IncludeClusterResources = true + BackupCfg.Selector = label + Expect(VeleroBackupNamespace(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, BackupCfg)).ShouldNot(HaveOccurred(), func() string { + VeleroBackupLogs(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, backupName) + return "Get backup logs" + }) + }) + + By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", VeleroCfg.StandbyCluster), func() { + Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.StandbyCluster)).To(Succeed()) + VeleroCfg.ObjectStoreProvider = "" + VeleroCfg.ClientToInstallVelero = VeleroCfg.StandbyClient + Expect(VeleroInstall(context.Background(), &VeleroCfg, false)).To(Succeed()) + }) + + By(fmt.Sprintf("Waiting for backups sync to Velero in cluster-B (%s)", VeleroCfg.StandbyCluster), func() { + Expect(WaitForBackupToBeCreated(context.Background(), VeleroCfg.VeleroCLI, backupName, 5*time.Minute)).To(Succeed()) + }) + + By(fmt.Sprintf("CRD %s should not exist in cluster-B (%s)", crdName, VeleroCfg.StandbyCluster), func() { + Expect(CRDShouldNotExist(context.Background(), crdName)).To(Succeed(), "Error: CRD already exists in cluster B, clean it and re-run test") + }) + + By("Restore CRD", func() { + Expect(VeleroRestore(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, restoreName, backupName, "")).To(Succeed(), func() string { + RunDebug(context.Background(), VeleroCfg.VeleroCLI, + VeleroCfg.VeleroNamespace, "", restoreName) + return "Fail to restore workload" + }) + }) + + By("Verify CRD restore ", func() { + Expect(CRDShouldExist(context.Background(), crdName)).To(Succeed()) + }) + }) + }) +} func APIGropuVersionsTest() { var ( resource, group string @@ -302,7 +424,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso } // Assertion - if containsAll(annoSpec["annotations"], tc.want["annotations"]) != true { + if !containsAll(annoSpec["annotations"], tc.want["annotations"]) { msg := fmt.Sprintf( "actual annotations: %v, expected annotations: %v", annoSpec["annotations"], @@ -312,7 +434,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso } // Assertion - if containsAll(annoSpec["specs"], tc.want["specs"]) != true { + if !containsAll(annoSpec["specs"], tc.want["specs"]) { msg := fmt.Sprintf( "actual specs: %v, expected specs: %v", annoSpec["specs"], @@ -367,6 +489,21 @@ func deleteCRD(ctx context.Context, yaml string) error { return nil } +func deleteCRDByName(ctx context.Context, name string) error { + fmt.Println("Delete CRD", name) + cmd := exec.CommandContext(ctx, "kubectl", "delete", "crd", name, "--wait") + + _, stderr, err := veleroexec.RunCommand(cmd) + if strings.Contains(stderr, "not found") { + return nil + } + if err != nil { + return errors.Wrap(err, stderr) + } + + return nil +} + func restartPods(ctx context.Context, ns string) error { fmt.Printf("Restart pods in %s namespace.\n", ns) cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns, "--wait=true") diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 9b3890c96a..9e509b3024 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -80,6 +80,7 @@ func init() { } var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", APIGropuVersionsTest) +var _ = Describe("[APIGroup][APIExtensions] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest) // Test backup and restore of Kibishi using restic var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) @@ -117,7 +118,6 @@ var _ = Describe("[BSL][Deletion][Snapshot] Local backups will be deleted once t var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) var _ = Describe("[Migration][Restic]", MigrationWithRestic) - var _ = Describe("[Migration][Snapshot]", MigrationWithSnapshots) var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 5c31ca9b78..90f96f578b 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -85,7 +85,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) DeleteNamespace(context.Background(), *VeleroCfg.StandbyClient, migrationNamespace, true) }) } - By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultClient), func() { + By(fmt.Sprintf("Switch to default kubeconfig context %s", VeleroCfg.DefaultCluster), func() { Expect(KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultCluster)).To(Succeed()) VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient }) diff --git a/test/e2e/testdata/enable_api_group_versions/case-a-source-v1beta1.yaml b/test/e2e/testdata/enable_api_group_versions/case-a-source-v1beta1.yaml new file mode 100644 index 0000000000..c72bb52004 --- /dev/null +++ b/test/e2e/testdata/enable_api_group_versions/case-a-source-v1beta1.yaml @@ -0,0 +1,90 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: music-system/music-serving-cert + controller-gen.kubebuilder.io/version: v0.2.5 + name: rocknrollbands.music.example.io +spec: + group: music.example.io + names: + kind: RocknrollBand + listKind: RocknrollBandList + plural: rocknrollbands + singular: rocknrollband + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: RocknrollBand is the Schema for the rocknrollbands API + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: RocknrollBandSpec defines the desired state of RocknrollBand + properties: + genre: + type: string + leadSinger: + type: string + numberComponents: + format: int32 + type: integer + type: object + status: + description: RocknrollBandStatus defines the observed state of RocknrollBand + properties: + lastPlayed: + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} + - name: v1alpha1 + schema: + openAPIV3Schema: + description: RocknrollBand is the Schema for the rocknrollbands API + properties: + apiVersion: + description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" + type: string + kind: + description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + type: string + metadata: + type: object + spec: + description: RocknrollBandSpec defines the desired state of RocknrollBand + properties: + genre: + type: string + numberComponents: + format: int32 + type: integer + type: object + status: + description: RocknrollBandStatus defines the observed state of RocknrollBand + properties: + lastPlayed: + type: string + required: + - lastPlayed + type: object + type: object + served: true + storage: false +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/test/e2e/util/k8s/common.go b/test/e2e/util/k8s/common.go index 6a378268cb..394a8d506f 100644 --- a/test/e2e/util/k8s/common.go +++ b/test/e2e/util/k8s/common.go @@ -49,7 +49,6 @@ func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace str data[key] = contents } - secret := builder.ForSecret(namespace, name).Data(data).Result() _, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) return err @@ -124,6 +123,45 @@ func GetPvByPvc(ctx context.Context, namespace, pvc string) ([]string, error) { return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) } +func CRDShouldExist(ctx context.Context, name string) error { + return CRDCountShouldBe(ctx, name, 1) +} + +func CRDShouldNotExist(ctx context.Context, name string) error { + return CRDCountShouldBe(ctx, name, 0) +} + +func CRDCountShouldBe(ctx context.Context, name string, count int) error { + crdList, err := GetCRD(ctx, name) + if err != nil { + return errors.Wrap(err, "Fail to get CRDs") + } + len := len(crdList) + if len != count { + return errors.New(fmt.Sprintf("CRD count is expected as %d instead of %d", count, len)) + } + return nil +} + +func GetCRD(ctx context.Context, name string) ([]string, error) { + CmdLine1 := &common.OsCommandLine{ + Cmd: "kubectl", + Args: []string{"get", "crd"}, + } + + CmdLine2 := &common.OsCommandLine{ + Cmd: "grep", + Args: []string{name}, + } + + CmdLine3 := &common.OsCommandLine{ + Cmd: "awk", + Args: []string{"{print $1}"}, + } + + return common.GetListBy2Pipes(ctx, *CmdLine1, *CmdLine2, *CmdLine3) +} + func AddLabelToPv(ctx context.Context, pv, label string) error { return exec.CommandContext(ctx, "kubectl", "label", "pv", pv, label).Run() } @@ -140,6 +178,12 @@ func AddLabelToPod(ctx context.Context, podName, namespace, label string) error return exec.CommandContext(ctx, "kubectl", args...).Run() } +func AddLabelToCRD(ctx context.Context, crd, label string) error { + args := []string{"label", "crd", crd, label} + fmt.Println(args) + return exec.CommandContext(ctx, "kubectl", args...).Run() +} + func KubectlApplyByFile(ctx context.Context, file string) error { args := []string{"apply", "-f", file, "--force=true"} return exec.CommandContext(ctx, "kubectl", args...).Run() @@ -154,3 +198,22 @@ func KubectlConfigUseContext(ctx context.Context, kubectlContext string) error { fmt.Print(stderr) return err } + +func GetAPIVersions(client *TestClient, name string) ([]string, error) { + var version []string + APIGroup, err := client.ClientGo.Discovery().ServerGroups() + if err != nil { + return nil, errors.Wrap(err, "Fail to get server API groups") + } + for _, group := range APIGroup.Groups { + fmt.Println(group.Name) + if group.Name == name { + for _, v := range group.Versions { + fmt.Println(v.Version) + version = append(version, v.Version) + } + return version, nil + } + } + return nil, errors.New("Server API groups is empty") +}