From 7afe70eae59c772c0f2c777725de12d68ee55a25 Mon Sep 17 00:00:00 2001 From: Carlisia Date: Wed, 17 Mar 2021 13:16:49 -0700 Subject: [PATCH 1/5] Code clean up Signed-off-by: Carlisia --- pkg/cmd/cli/install/install.go | 5 +---- pkg/install/install.go | 11 +++++++---- pkg/install/resources.go | 4 ++-- test/e2e/enable_api_group_versions_test.go | 2 +- test/e2e/velero_utils.go | 14 ++++++++------ 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index eb82cbbfd8..d2ed61c6af 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -255,10 +255,7 @@ func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error { return err } - resources, err = install.AllResources(vo) - if err != nil { - return err - } + resources = install.AllResources(vo) } if _, err := output.PrintWithFormat(c, resources); err != nil { diff --git a/pkg/install/install.go b/pkg/install/install.go index 88e12a71a0..7d1871ed51 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -234,16 +234,19 @@ func createResource(r *unstructured.Unstructured, factory client.DynamicFactory, format := strings.Join([]string{id, ": ", f, "\n"}, "") fmt.Fprintf(w, format, a...) } - log("attempting to create resource") c, err := CreateClient(r, factory, w) if err != nil { return err } - if _, err := c.Create(r); apierrors.IsAlreadyExists(err) { - log("already exists, proceeding") - } else if err != nil { + log("attempting to create resource") + _, err = c.Create(r) + if err != nil { + if apierrors.IsAlreadyExists(err) { + log("already exists, proceeding") + return nil + } return errors.Wrapf(err, "Error creating resource %s", id) } diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 885232045f..9ca6393881 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -245,7 +245,7 @@ func AllCRDs() *unstructured.UnstructuredList { // AllResources returns a list of all resources necessary to install Velero, in the appropriate order, into a Kubernetes cluster. // Items are unstructured, since there are different data types returned. -func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) { +func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { resources := AllCRDs() ns := Namespace(o.Namespace) @@ -317,5 +317,5 @@ func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) { appendUnstructured(resources, ds) } - return resources, nil + return resources } diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 20be75bf6a..c305a081e7 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -380,7 +380,7 @@ func installVeleroForAPIGroups(ctx context.Context) error { options.Image = veleroImage if err := InstallVeleroServer(options); err != nil { - return errors.Wrap(err, "install velero server") + return errors.WithMessagef(err, "Failed to install Velero in the cluster") } return nil diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index d23d969e21..d40482fb19 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -92,17 +92,17 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error { } f := client.NewFactory("e2e", config) - resources, err := install.AllResources(vo) - if err != nil { - return errors.Wrap(err, "Failed to install Velero in the cluster") - } + resources := install.AllResources(vo) dynamicClient, err := f.DynamicClient() if err != nil { return err } + factory := client.NewDynamicFactory(dynamicClient) + errorMsg := "\n\nError installing Velero. Use `kubectl logs deploy/velero -n velero` to check the deploy logs" + err = install.Install(factory, resources, os.Stdout) if err != nil { return errors.Wrap(err, errorMsg) @@ -119,6 +119,7 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error { return errors.Wrap(err, errorMsg) } } + fmt.Printf("Velero is installed! ⛵ Use 'kubectl logs deployment/velero -n %s' to view the status.\n", io.Namespace) return nil } @@ -274,7 +275,7 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri } err := EnsureClusterExists(ctx) if err != nil { - return errors.WithMessage(err, "Failed to ensure kubernetes cluster exists") + return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } veleroInstallOptions, err := GetProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, getProviderPlugins(objectStoreProvider), features) @@ -286,8 +287,9 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri veleroInstallOptions.Namespace = veleroNamespace err = InstallVeleroServer(veleroInstallOptions) if err != nil { - return errors.WithMessagef(err, "Failed to install Velero in cluster") + return errors.WithMessagef(err, "Failed to install Velero in the cluster") } + return nil } From 2c30c0442e461f909cc772974c17b078a3fc4a93 Mon Sep 17 00:00:00 2001 From: Carlisia Date: Fri, 19 Mar 2021 18:49:10 -0700 Subject: [PATCH 2/5] Fix uninstall + left over resources after test failure Signed-off-by: Carlisia --- pkg/client/factory.go | 7 +- pkg/cmd/cli/uninstall/uninstall.go | 144 ++++++++---- pkg/util/kube/utils.go | 26 --- test/e2e/README.md | 5 + test/e2e/backup_test.go | 24 +- test/e2e/client.go | 61 +++++ test/e2e/common.go | 23 +- test/e2e/enable_api_group_versions_test.go | 252 +++++++++++++-------- test/e2e/kibishii_tests.go | 9 +- test/e2e/velero_utils.go | 40 ++-- 10 files changed, 362 insertions(+), 229 deletions(-) create mode 100644 test/e2e/client.go diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 6ecc63be74..5d55ccedd6 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -23,9 +23,11 @@ import ( "github.com/pkg/errors" "github.com/spf13/pflag" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + k8scheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -45,7 +47,8 @@ type Factory interface { // DynamicClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. DynamicClient() (dynamic.Interface, error) - // KubebuilderClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster + // KubebuilderClient returns a client for the controller runtime framework. It adds Kubernetes and Velero + // types to its scheme. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. KubebuilderClient() (kbclient.Client, error) // SetBasename changes the basename for an already-constructed client. @@ -151,6 +154,8 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) + k8scheme.AddToScheme(scheme) + apiextv1beta1.AddToScheme(scheme) kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index f7ce824cad..910a8deddf 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -19,57 +19,126 @@ package uninstall import ( "context" "fmt" - "strings" "time" "github.com/pkg/errors" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + + // apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/install" - "github.com/vmware-tanzu/velero/pkg/util/kube" ) -// Uninstall uninstalls all components deployed using velero install command -func Uninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclientset.Clientset, veleroNamespace string) error { - if veleroNamespace == "" { - veleroNamespace = "velero" +// NewCommand creates a cobra command. +func NewCommand(f client.Factory) *cobra.Command { + c := &cobra.Command{ + Use: "uninstall", + Short: "Uninstall Velero", + Long: `Uninstall Velero along with the CRDs. + +The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero). + `, + Example: `# velero uninstall -n backup`, + Run: func(c *cobra.Command, args []string) { + + if !cli.GetConfirmation() { + // Don't do anything unless we get confirmation + return + } + + kbClient, err := f.KubebuilderClient() + cmd.CheckError(err) + cmd.CheckError(Run(kbClient, f.Namespace())) + }, } - err := DeleteNamespace(ctx, client, veleroNamespace) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing Velero namespace %s", veleroNamespace) + + output.BindFlags(c.Flags()) + output.ClearOutputFlagDefault(c) + return c +} + +// Run removes all components that were deployed using the Velero install command +func Run(kbClient kbclient.Client, namespace string) error { + var errs []error + + // namespace + ns := &corev1.Namespace{} + key := kbclient.ObjectKey{Name: namespace} + if err := kbClient.Get(context.Background(), key, ns); err != nil { + if apierrors.IsNotFound(err) { + fmt.Printf("Velero installation namespace %q does not exist, skipping.\n", namespace) + } else { + errs = append(errs, errors.WithStack(err)) + } + } else { + if ns.Status.Phase == corev1.NamespaceTerminating { + fmt.Printf("Velero installation namespace %q is terminating.\n", namespace) + } else { + if err := kbClient.Delete(context.Background(), ns); err != nil { + errs = append(errs, errors.WithStack(err)) + } + } } - rolebinding := install.ClusterRoleBinding(veleroNamespace) + time.Sleep(time.Second * 60) - err = client.RbacV1().ClusterRoleBindings().Delete(ctx, rolebinding.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing Velero cluster role binding %s", rolebinding) + // rolebinding + crb := install.ClusterRoleBinding(namespace) + key = kbclient.ObjectKey{Name: crb.Name, Namespace: namespace} + if err := kbClient.Get(context.Background(), key, crb); err != nil { + if apierrors.IsNotFound(err) { + fmt.Printf("Velero installation rolebinding %q does not exist, skipping.\n", crb.Name) + } else { + errs = append(errs, errors.WithStack(err)) + } + } else { + if err := kbClient.Delete(context.Background(), crb); err != nil { + errs = append(errs, errors.WithStack(err)) + } } - veleroLabels := labels.FormatLabels(install.Labels()) - crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{ - LabelSelector: veleroLabels, - }) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed listing Velero crds") + // CRDs + veleroLabels := labels.FormatLabels(install.Labels()) + crdList := apiextv1beta1.CustomResourceDefinitionList{} + opts := kbclient.ListOptions{ + Namespace: namespace, + Raw: &metav1.ListOptions{ + LabelSelector: veleroLabels, + }, } - for _, removeCRD := range crds.Items { - err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name) + if err := kbClient.List(context.Background(), &crdList, &opts); err != nil { + errs = append(errs, errors.WithStack(err)) + } else { + if len(crdList.Items) == 0 { + fmt.Print("Velero CRDs do not exist, skipping.\n") + } else { + for _, crd := range crdList.Items { + if err := kbClient.Delete(context.Background(), &crd); err != nil { + errs = append(errs, errors.WithStack(err)) + } + } } } - fmt.Println("Uninstalled Velero") + if kubeerrs.NewAggregate(errs) != nil { + fmt.Printf("Errors while attempting to uninstall Velero: %q", kubeerrs.NewAggregate(errs)) + return kubeerrs.NewAggregate(errs) + } + + fmt.Println("Velero uninstalled ⛵") return nil } @@ -89,26 +158,3 @@ func DeleteNamespace(ctx context.Context, client *kubernetes.Clientset, namespac return false, nil }) } - -// NewCommand creates a cobra command. -func NewCommand(f client.Factory) *cobra.Command { - c := &cobra.Command{ - Use: "uninstall", - Short: "Uninstall Velero", - Long: `Uninstall Velero along with the CRDs. - -The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero). - `, - Example: `# velero uninstall -n backup`, - Run: func(c *cobra.Command, args []string) { - veleroNs := strings.TrimSpace(f.Namespace()) - cl, extCl, err := kube.GetClusterClient() - cmd.CheckError(err) - cmd.CheckError(Uninstall(context.Background(), cl, extCl, veleroNs)) - }, - } - - output.BindFlags(c.Flags()) - output.ClearOutputFlagDefault(c) - return c -} diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index c8262080d3..994e8f370c 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -24,15 +24,12 @@ import ( "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/clientcmd" ) // NamespaceAndName returns a string in the format / @@ -220,26 +217,3 @@ func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) { return (isEstablished && namesAccepted), nil } - -// GetClusterClient instantiates and returns a client for the cluster. -func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) { - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - configOverrides := &clientcmd.ConfigOverrides{} - kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) - clientConfig, err := kubeConfig.ClientConfig() - if err != nil { - return nil, nil, errors.WithStack(err) - } - - client, err := kubernetes.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - return client, extensionClientSet, nil -} diff --git a/test/e2e/README.md b/test/e2e/README.md index 08eeb40c32..013fb5cb3a 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -105,3 +105,8 @@ For example, E2E tests can be run from Velero repository roots using the command Velero E2E tests uses [Ginkgo](https://onsi.github.io/ginkgo/) testing framework which allows a subset of the tests to be run using the [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) flags to ginkgo. The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` make variable. This can be used to focus on specific tests. + +## Adding tests + +### Tips +Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 20dc96ecb9..363455b470 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -4,15 +4,10 @@ import ( "context" "flag" "fmt" - "time" "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - "k8s.io/client-go/kubernetes" - - "github.com/vmware-tanzu/velero/pkg/util/kube" ) var ( @@ -34,12 +29,13 @@ func backup_restore_with_restic() { func backup_restore_test(useVolumeSnapshots bool) { var ( - client *kubernetes.Clientset - extensionsClient *apiextensionsclientset.Clientset - backupName string - restoreName string + backupName string + restoreName string ) + client, err := NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + BeforeEach(func() { if useVolumeSnapshots && cloudProvider == "kind" { Skip("Volume snapshots not supported on kind") @@ -53,17 +49,11 @@ func backup_restore_test(useVolumeSnapshots bool) { cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")).To(Succeed()) } - client, extensionsClient, err = kube.GetClusterClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client") }) AfterEach(func() { - if installVelero { - timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute) - err := VeleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace) - Expect(err).To(Succeed()) - } - + err = VeleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) + Expect(err).To(Succeed()) }) When("kibishii is the sample workload", func() { diff --git a/test/e2e/client.go b/test/e2e/client.go new file mode 100644 index 0000000000..eaf31cdfa4 --- /dev/null +++ b/test/e2e/client.go @@ -0,0 +1,61 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "github.com/vmware-tanzu/velero/pkg/client" + "k8s.io/client-go/kubernetes" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +type TestClient struct { + Kubebuilder kbclient.Client + ClientGo kubernetes.Interface + DynamicFactory client.DynamicFactory +} + +func NewTestClient() (TestClient, error) { + config, err := client.LoadConfig() + if err != nil { + return TestClient{}, err + } + + f := client.NewFactory("e2e", config) + + clientGo, err := f.KubeClient() + if err != nil { + return TestClient{}, err + } + + kb, err := f.KubebuilderClient() + if err != nil { + return TestClient{}, err + } + + dynamicClient, err := f.DynamicClient() + if err != nil { + return TestClient{}, err + } + + factory := client.NewDynamicFactory(dynamicClient) + + return TestClient{ + Kubebuilder: kb, + ClientGo: clientGo, + DynamicFactory: factory, + }, nil +} diff --git a/test/e2e/common.go b/test/e2e/common.go index 6dbfc02914..9b348bc3d6 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -13,7 +13,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/builder" ) @@ -24,19 +23,19 @@ func EnsureClusterExists(ctx context.Context) error { } // CreateNamespace creates a kubernetes namespace -func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { +func CreateNamespace(ctx context.Context, client TestClient, namespace string) error { ns := builder.ForNamespace(namespace).Result() - _, err := client.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } return err } -// WaitForNamespaceDeletion Waits for namespace to be deleted. -func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernetes.Clientset, ns string) error { +// WaitForNamespaceDeletion waits for namespace to be deleted. +func WaitForNamespaceDeletion(interval, timeout time.Duration, client TestClient, ns string) error { err := wait.PollImmediate(interval, timeout, func() (bool, error) { - _, err := client.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil @@ -49,7 +48,7 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete return err } -func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error { +func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace string, name string, files map[string]string) error { data := make(map[string][]byte) for key, filePath := range files { @@ -62,19 +61,17 @@ func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, na } secret := builder.ForSecret(namespace, name).Data(data).Result() - _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) return err } -/* - Waits until all of the pods have gone to PodRunning state -*/ -func WaitForPods(ctx context.Context, client *kubernetes.Clientset, namespace string, pods []string) error { +// WaitForPods waits until all of the pods have gone to PodRunning state +func WaitForPods(ctx context.Context, client TestClient, namespace string, pods []string) error { timeout := 10 * time.Minute interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { for _, podName := range pods { - checkPod, err := client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) if err != nil { return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning)) } diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index c305a081e7..d14ee0ac14 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -12,17 +12,12 @@ import ( "github.com/google/uuid" - "github.com/vmware-tanzu/velero/pkg/util/kube" - - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/builder" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" @@ -30,26 +25,28 @@ import ( var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() { var ( - resource, group string - certMgrCRD map[string]string - client *kubernetes.Clientset - extensionsClient *apiextensionsclient.Clientset - err error - ctx = context.Background() + resource, group string + certManagerCRD map[string]resourceCRD + err error + ctx = context.Background() ) + const cert = "cert" + + client, err := NewTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") BeforeEach(func() { resource = "rockbands" group = "music.example.io" - certMgrCRD = map[string]string{ - "url": "testdata/enable_api_group_versions/cert-manager.yaml", - "namespace": "cert-manager", - } - - client, extensionsClient, err = kube.GetClusterClient() // Currently we ignore the API extensions client - Expect(err).NotTo(HaveOccurred()) + certManagerCRD = + map[string]resourceCRD{ + cert: resourceCRD{ + url: "testdata/enable_api_group_versions/cert-manager.yaml", + ns: "cert-manager", + }, + } - err = InstallCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) + err = InstallCRD(ctx, certManagerCRD[cert]) Expect(err).NotTo(HaveOccurred()) uuidgen, err = uuid.NewRandom() @@ -63,30 +60,31 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", cmd = exec.CommandContext(ctx, "kubectl", "delete", "crd", "rockbands.music.example.io") _, _, _ = veleroexec.RunCommand(cmd) - _ = DeleteCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) + _ = DeleteCRD(ctx, certManagerCRD[cert]) }) Context("When EnableAPIGroupVersions flag is set", func() { It("Should back up API group version and restore by version priority", func() { Expect(RunEnableAPIGroupVersionsTests( ctx, + client, resource, group, - client, - extensionsClient, + certManagerCRD[cert], )).To(Succeed(), "Failed to successfully backup and restore multiple API Groups") }) }) }) -func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset, - extensionsClient *apiextensionsclient.Clientset) error { +func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, resource, group string, certManager resourceCRD) error { + const src = "source" + const tgt = "target" + tests := []struct { name string namespaces []string - srcCRD map[string]string + resources map[string]resourceCRD srcCRs map[string]string - tgtCRD map[string]string tgtVer string cm *corev1api.ConfigMap gvs map[string][]string @@ -94,18 +92,20 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, }{ { name: "Target and source cluster preferred versions match; Preferred version v1 is restored (Priority 1, Case A).", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-a-source.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: resourceCRD{ + url: "testdata/enable_api_group_versions/case-a-source.yaml", + ns: "music-system", + }, + tgt: resourceCRD{ + url: "testdata/enable_api_group_versions/case-a-target.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", "v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-a-target.yaml", - "namespace": "music-system", - }, tgtVer: "v1", cm: nil, want: map[string]map[string]string{ @@ -120,19 +120,21 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, }, { name: "Latest common non-preferred supported version v2beta2 is restored (Priority 3, Case D).", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: resourceCRD{ + url: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", + ns: "music-system", + }, + tgt: resourceCRD{ + url: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml", "v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml", "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", - "namespace": "music-system", - }, tgtVer: "v2beta2", cm: nil, want: map[string]map[string]string{ @@ -148,37 +150,41 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, }, { name: "No common supported versions means no rockbands custom resource is restored.", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-a-source.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: resourceCRD{ + url: "testdata/enable_api_group_versions/case-a-source.yaml", + ns: "music-system", + }, + tgt: resourceCRD{ + url: "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", "v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml", - "namespace": "music-system", - }, tgtVer: "", cm: nil, want: nil, }, { name: "User config map overrides Priority 3, Case D and restores v2beta1", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: resourceCRD{ + url: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", + ns: "music-system", + }, + tgt: resourceCRD{ + url: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml", "v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml", "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", - "namespace": "music-system", - }, tgtVer: "v2beta1", cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data( "restoreResourcesVersionPriority", @@ -200,23 +206,31 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, for i, tc := range tests { fmt.Printf("\n====== Test Case %d ======\n", i) - err := InstallCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]) + err := InstallCRD(ctx, tc.resources[src]) if err != nil { return errors.Wrap(err, "installing music-system CRD for source cluster") } for version, cr := range tc.srcCRs { ns := resource + "-src-" + version + tc.namespaces = append(tc.namespaces, ns) if err := CreateNamespace(ctx, client, ns); err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "creating %s namespace", ns) } if err := InstallCR(ctx, cr, ns); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "installing %s custom resource on source cluster namespace %s", cr, ns) } - tc.namespaces = append(tc.namespaces, ns) } // TODO - Velero needs to be installed AFTER CRDs are installed because of https://github.com/vmware-tanzu/velero/issues/3471 @@ -235,16 +249,27 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) if err != nil { VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr) } // Delete music-system CRD and controllers installed on source cluster. - if err := DeleteCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]); err != nil { + if err := DeleteCRD(ctx, tc.resources[src]); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + }) return errors.Wrapf(err, "deleting music-system CRD from source cluster") } for _, ns := range tc.namespaces { - if err := client.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { + if err := client.ClientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } @@ -254,20 +279,34 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, } // Install music-system CRD for target cluster. - if err := InstallCRD(ctx, tc.tgtCRD["url"], tc.tgtCRD["namespace"]); err != nil { + if err := InstallCRD(ctx, tc.resources[tgt]); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "installing music-system CRD for target cluster") } // Apply config map if there is one. if tc.cm != nil { - _, err := client.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrap(err, "creating config map with user version priorities") } } // Reset Velero to recognize music-system CRD. if err := RestartPods(ctx, veleroNamespace); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrapf(err, "restarting Velero pods") } fmt.Println("Sleep 20s to wait for Velero to stabilize after restart.") @@ -279,11 +318,20 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, if tc.want != nil { if err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { VeleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrapf(err, "restoring %s namespaces on target cluster", namespacesStr) } annoSpec, err := resourceInfo(ctx, group, tc.tgtVer, resource) if err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrapf( err, "get annotation and spec from %s.%s/%s object", @@ -300,6 +348,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, annoSpec["annotations"], tc.want["annotations"], ) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.New(msg) } @@ -310,6 +363,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, annoSpec["specs"], tc.want["specs"], ) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.New(msg) } } else { @@ -318,43 +376,47 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.New("expected error but not none") } } - // Delete namespaces created for CRs - for _, ns := range tc.namespaces { - fmt.Println("Delete namespace", ns) - _ = client.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) - _ = WaitNamespaceDelete(ctx, ns) - } + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) - // Delete source cluster music-system CRD - _ = DeleteCRD( - ctx, - tc.srcCRD["url"], - tc.srcCRD["namespace"], - ) - - // Delete target cluster music-system CRD - _ = DeleteCRD( - ctx, - tc.tgtCRD["url"], - tc.srcCRD["namespace"], - ) - - // Uninstall Velero - if installVelero { - err = VeleroUninstall(ctx, client, extensionsClient, veleroNamespace) - if err != nil { - return err - } + err = VeleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) + if err != nil { + return err } } return nil } +type resourceCRD struct { + url, ns string +} + +func testCleanup(ctx context.Context, client TestClient, namespaces []string, resources []resourceCRD) { + // Delete namespaces created for CRs + for _, ns := range namespaces { + fmt.Println("Delete namespace", ns) + _ = client.ClientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + _ = WaitNamespaceDelete(ctx, ns) + } + + for _, rs := range resources { + DeleteCRD(ctx, rs) + } +} + func installVeleroForAPIGroups(ctx context.Context) error { if err := EnsureClusterExists(ctx); err != nil { return errors.Wrap(err, "check cluster exists") @@ -386,17 +448,17 @@ func installVeleroForAPIGroups(ctx context.Context) error { return nil } -func InstallCRD(ctx context.Context, crdFile, ns string) error { - fmt.Printf("Install CRD %s.\n", crdFile) +func InstallCRD(ctx context.Context, crd resourceCRD) error { + fmt.Printf("Install CRD %s.\n", crd.url) - cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", crdFile) + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", crd.url) _, stderr, err := veleroexec.RunCommand(cmd) if err != nil { return errors.Wrap(err, stderr) } fmt.Println("Wait for CRD to be ready.") - if err := WaitForPodContainers(ctx, ns); err != nil { + if err := WaitForPodContainers(ctx, crd.ns); err != nil { return err } @@ -436,9 +498,9 @@ func WaitForPodContainers(ctx context.Context, ns string) error { return err } -func DeleteCRD(ctx context.Context, crdFile, ns string) error { - fmt.Println("Delete CRD", crdFile) - cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crdFile, "--wait") +func DeleteCRD(ctx context.Context, crd resourceCRD) error { + fmt.Println("Delete CRD", crd.url) + cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crd.url, "--wait") _, stderr, err := veleroexec.RunCommand(cmd) if strings.Contains(stderr, "not found") { @@ -450,7 +512,7 @@ func DeleteCRD(ctx context.Context, crdFile, ns string) error { } err = wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { - cmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", ns) + cmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", crd.ns) stdout, stderr, err := veleroexec.RunCommand(cmd) if strings.Contains(stderr, "not found") { @@ -461,7 +523,7 @@ func DeleteCRD(ctx context.Context, crdFile, ns string) error { return false, errors.Wrap(err, stderr) } - re := regexp.MustCompile(ns) + re := regexp.MustCompile(crd.ns) return re.MatchString(stdout), nil }) return err diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 624be7ad01..b7b423a1d2 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -8,7 +8,6 @@ import ( "github.com/pkg/errors" "golang.org/x/net/context" - "k8s.io/client-go/kubernetes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -78,7 +77,7 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel } // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, +func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, useVolumeSnapshots bool) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) @@ -110,7 +109,7 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel } fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.ClientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, "Failed to simulate a disaster") } // wait for ns delete @@ -137,7 +136,7 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrap(err, "Failed to verify data generated by kibishii") } - if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.ClientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace) } // wait for ns delete @@ -148,6 +147,6 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return nil } -func waitForKibishiiPods(ctx context.Context, client *kubernetes.Clientset, kibishiiNamespace string) error { +func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespace string) error { return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index d40482fb19..02c26bceb7 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -12,12 +12,9 @@ import ( "strings" "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" - - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/client" cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/cli/uninstall" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" @@ -81,45 +78,36 @@ func GetProviderVeleroInstallOptions( // InstallVeleroServer installs velero in the cluster. func InstallVeleroServer(io *cliinstall.InstallOptions) error { - config, err := client.LoadConfig() - if err != nil { - return err - } - vo, err := io.AsVeleroOptions() if err != nil { return errors.Wrap(err, "Failed to translate InstallOptions to VeleroOptions for Velero") } - f := client.NewFactory("e2e", config) - resources := install.AllResources(vo) - - dynamicClient, err := f.DynamicClient() + client, err := NewTestClient() if err != nil { - return err + return errors.Wrap(err, "Failed to instantiate cluster client for installing Velero") } - factory := client.NewDynamicFactory(dynamicClient) - errorMsg := "\n\nError installing Velero. Use `kubectl logs deploy/velero -n velero` to check the deploy logs" - - err = install.Install(factory, resources, os.Stdout) + resources := install.AllResources(vo) + err = install.Install(client.DynamicFactory, resources, os.Stdout) if err != nil { return errors.Wrap(err, errorMsg) } fmt.Println("Waiting for Velero deployment to be ready.") - if _, err = install.DeploymentIsReady(factory, io.Namespace); err != nil { + if _, err = install.DeploymentIsReady(client.DynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } if io.UseRestic { fmt.Println("Waiting for Velero restic daemonset to be ready.") - if _, err = install.DaemonSetIsReady(factory, io.Namespace); err != nil { + if _, err = install.DaemonSetIsReady(client.DynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } } - fmt.Printf("Velero is installed! ⛵ Use 'kubectl logs deployment/velero -n %s' to view the status.\n", io.Namespace) + + fmt.Printf("Velero is installed and ready to be tested in the %s namespace! ⛵ \n", io.Namespace) return nil } @@ -293,8 +281,14 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri return nil } -func VeleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, veleroNamespace string) error { - return uninstall.Uninstall(ctx, client, extensionsClient, veleroNamespace) +func VeleroUninstall(client kbclient.Client, installVelero bool, veleroNamespace string) error { + if installVelero { + err := uninstall.Run(client, veleroNamespace) + if err != nil { + return err + } + } + return nil } func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { From fd51611df52a1d99103e5df2efbd06afe8bbe126 Mon Sep 17 00:00:00 2001 From: Carlisia Date: Fri, 19 Mar 2021 20:48:17 -0700 Subject: [PATCH 3/5] Unexport code and add copyright Signed-off-by: Carlisia --- test/e2e/backup_test.go | 28 ++++-- test/e2e/client.go | 14 +-- test/e2e/common.go | 34 ++++++-- test/e2e/e2e_suite_test.go | 16 ++++ test/e2e/enable_api_group_versions_test.go | 99 +++++++++++++--------- test/e2e/kibishii_tests.go | 36 +++++--- test/e2e/velero_utils.go | 64 ++++++++------ 7 files changed, 194 insertions(+), 97 deletions(-) diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 363455b470..ead5cacea7 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -33,7 +49,7 @@ func backup_restore_test(useVolumeSnapshots bool) { restoreName string ) - client, err := NewTestClient() + client, err := newTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") BeforeEach(func() { @@ -45,14 +61,14 @@ func backup_restore_test(useVolumeSnapshots bool) { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")).To(Succeed()) } }) AfterEach(func() { - err = VeleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) + err = veleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) Expect(err).To(Succeed()) }) @@ -79,7 +95,7 @@ func backup_restore_test(useVolumeSnapshots bool) { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) + Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen) @@ -88,11 +104,11 @@ func backup_restore_test(useVolumeSnapshots bool) { secretKey: additionalBSLCredentials, } - Expect(CreateSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) + Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) // Create additional BSL using credential additionalBsl := fmt.Sprintf("bsl-%s", uuidgen) - Expect(VeleroCreateBackupLocation(context.TODO(), + Expect(veleroCreateBackupLocation(context.TODO(), veleroCLI, veleroNamespace, additionalBsl, diff --git a/test/e2e/client.go b/test/e2e/client.go index eaf31cdfa4..2ab5d54906 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -22,38 +22,38 @@ import ( kbclient "sigs.k8s.io/controller-runtime/pkg/client" ) -type TestClient struct { +type testClient struct { Kubebuilder kbclient.Client ClientGo kubernetes.Interface DynamicFactory client.DynamicFactory } -func NewTestClient() (TestClient, error) { +func newTestClient() (testClient, error) { config, err := client.LoadConfig() if err != nil { - return TestClient{}, err + return testClient{}, err } f := client.NewFactory("e2e", config) clientGo, err := f.KubeClient() if err != nil { - return TestClient{}, err + return testClient{}, err } kb, err := f.KubebuilderClient() if err != nil { - return TestClient{}, err + return testClient{}, err } dynamicClient, err := f.DynamicClient() if err != nil { - return TestClient{}, err + return testClient{}, err } factory := client.NewDynamicFactory(dynamicClient) - return TestClient{ + return testClient{ Kubebuilder: kb, ClientGo: clientGo, DynamicFactory: factory, diff --git a/test/e2e/common.go b/test/e2e/common.go index 9b348bc3d6..c6632335a4 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -17,13 +33,13 @@ import ( "github.com/vmware-tanzu/velero/pkg/builder" ) -// EnsureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. -func EnsureClusterExists(ctx context.Context) error { +// ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. +func ensureClusterExists(ctx context.Context) error { return exec.CommandContext(ctx, "kubectl", "cluster-info").Run() } -// CreateNamespace creates a kubernetes namespace -func CreateNamespace(ctx context.Context, client TestClient, namespace string) error { +// createNamespace creates a kubernetes namespace +func createNamespace(ctx context.Context, client testClient, namespace string) error { ns := builder.ForNamespace(namespace).Result() _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { @@ -32,8 +48,8 @@ func CreateNamespace(ctx context.Context, client TestClient, namespace string) e return err } -// WaitForNamespaceDeletion waits for namespace to be deleted. -func WaitForNamespaceDeletion(interval, timeout time.Duration, client TestClient, ns string) error { +// waitForNamespaceDeletion waits for namespace to be deleted. +func waitForNamespaceDeletion(interval, timeout time.Duration, client testClient, ns string) error { err := wait.PollImmediate(interval, timeout, func() (bool, error) { _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) if err != nil { @@ -48,7 +64,7 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client TestClient return err } -func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace string, name string, files map[string]string) error { +func createSecretFromFiles(ctx context.Context, client testClient, namespace string, name string, files map[string]string) error { data := make(map[string][]byte) for key, filePath := range files { @@ -65,8 +81,8 @@ func CreateSecretFromFiles(ctx context.Context, client TestClient, namespace str return err } -// WaitForPods waits until all of the pods have gone to PodRunning state -func WaitForPods(ctx context.Context, client TestClient, namespace string, pods []string) error { +// waitForPods waits until all of the pods have gone to PodRunning state +func waitForPods(ctx context.Context, client testClient, namespace string, pods []string) error { timeout := 10 * time.Minute interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ecac96bd4c..0429070ec3 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index d14ee0ac14..d9e6bbeb99 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -32,7 +48,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", ) const cert = "cert" - client, err := NewTestClient() + client, err := newTestClient() Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") BeforeEach(func() { @@ -46,7 +62,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", }, } - err = InstallCRD(ctx, certManagerCRD[cert]) + err = installCRD(ctx, certManagerCRD[cert]) Expect(err).NotTo(HaveOccurred()) uuidgen, err = uuid.NewRandom() @@ -60,7 +76,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", cmd = exec.CommandContext(ctx, "kubectl", "delete", "crd", "rockbands.music.example.io") _, _, _ = veleroexec.RunCommand(cmd) - _ = DeleteCRD(ctx, certManagerCRD[cert]) + _ = deleteCRD(ctx, certManagerCRD[cert]) }) Context("When EnableAPIGroupVersions flag is set", func() { @@ -76,9 +92,10 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", }) }) -func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, resource, group string, certManager resourceCRD) error { +func RunEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string, certManager resourceCRD) error { const src = "source" const tgt = "target" + const srcCRs = "srcCRs" tests := []struct { name string @@ -93,11 +110,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso { name: "Target and source cluster preferred versions match; Preferred version v1 is restored (Priority 1, Case A).", resources: map[string]resourceCRD{ - src: resourceCRD{ + src: { url: "testdata/enable_api_group_versions/case-a-source.yaml", ns: "music-system", }, - tgt: resourceCRD{ + tgt: { url: "testdata/enable_api_group_versions/case-a-target.yaml", ns: "music-system", }, @@ -121,11 +138,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso { name: "Latest common non-preferred supported version v2beta2 is restored (Priority 3, Case D).", resources: map[string]resourceCRD{ - src: resourceCRD{ + src: { url: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", ns: "music-system", }, - tgt: resourceCRD{ + tgt: { url: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", ns: "music-system", }, @@ -151,11 +168,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso { name: "No common supported versions means no rockbands custom resource is restored.", resources: map[string]resourceCRD{ - src: resourceCRD{ + src: { url: "testdata/enable_api_group_versions/case-a-source.yaml", ns: "music-system", }, - tgt: resourceCRD{ + tgt: { url: "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml", ns: "music-system", }, @@ -171,11 +188,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso { name: "User config map overrides Priority 3, Case D and restores v2beta1", resources: map[string]resourceCRD{ - src: resourceCRD{ + src: { url: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", ns: "music-system", }, - tgt: resourceCRD{ + tgt: { url: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", ns: "music-system", }, @@ -206,7 +223,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso for i, tc := range tests { fmt.Printf("\n====== Test Case %d ======\n", i) - err := InstallCRD(ctx, tc.resources[src]) + err := installCRD(ctx, tc.resources[src]) if err != nil { return errors.Wrap(err, "installing music-system CRD for source cluster") } @@ -215,7 +232,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso ns := resource + "-src-" + version tc.namespaces = append(tc.namespaces, ns) - if err := CreateNamespace(ctx, client, ns); err != nil { + if err := createNamespace(ctx, client, ns); err != nil { testCleanup(ctx, client, nil, []resourceCRD{ certManager, tc.resources[src], @@ -223,7 +240,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso return errors.Wrapf(err, "creating %s namespace", ns) } - if err := InstallCR(ctx, cr, ns); err != nil { + if err := installCR(ctx, cr, ns); err != nil { testCleanup(ctx, client, tc.namespaces, []resourceCRD{ certManager, tc.resources[src], @@ -236,7 +253,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso // TODO - Velero needs to be installed AFTER CRDs are installed because of https://github.com/vmware-tanzu/velero/issues/3471 // Once that issue is fixed, we should install Velero once for the test suite if installVelero { - VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, + veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "EnableAPIGroupVersions" /* TODO - remove this when the feature flag is removed */) fmt.Println("Sleep 20s to wait for Velero to stabilize after install.") @@ -246,9 +263,9 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) + err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) if err != nil { - VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) + veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) testCleanup(ctx, client, tc.namespaces, []resourceCRD{ certManager, tc.resources[src], @@ -257,7 +274,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso } // Delete music-system CRD and controllers installed on source cluster. - if err := DeleteCRD(ctx, tc.resources[src]); err != nil { + if err := deleteCRD(ctx, tc.resources[src]); err != nil { testCleanup(ctx, client, tc.namespaces, []resourceCRD{ certManager, }) @@ -273,13 +290,13 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } - if err := WaitNamespaceDelete(ctx, ns); err != nil { + if err := waitNamespaceDelete(ctx, ns); err != nil { return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } } // Install music-system CRD for target cluster. - if err := InstallCRD(ctx, tc.resources[tgt]); err != nil { + if err := installCRD(ctx, tc.resources[tgt]); err != nil { testCleanup(ctx, client, tc.namespaces, []resourceCRD{ certManager, tc.resources[src], @@ -301,7 +318,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso } // Reset Velero to recognize music-system CRD. - if err := RestartPods(ctx, veleroNamespace); err != nil { + if err := restartPods(ctx, veleroNamespace); err != nil { testCleanup(ctx, client, tc.namespaces, []resourceCRD{ certManager, tc.resources[src], @@ -316,8 +333,8 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { - VeleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) + if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { + veleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) testCleanup(ctx, client, tc.namespaces, []resourceCRD{ tc.resources[src], tc.resources[tgt], @@ -373,7 +390,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) + err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { testCleanup(ctx, client, tc.namespaces, []resourceCRD{ @@ -391,7 +408,7 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client TestClient, reso tc.resources[tgt], }) - err = VeleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) + err = veleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) if err != nil { return err } @@ -404,26 +421,26 @@ type resourceCRD struct { url, ns string } -func testCleanup(ctx context.Context, client TestClient, namespaces []string, resources []resourceCRD) { +func testCleanup(ctx context.Context, client testClient, namespaces []string, resources []resourceCRD) { // Delete namespaces created for CRs for _, ns := range namespaces { fmt.Println("Delete namespace", ns) _ = client.ClientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) - _ = WaitNamespaceDelete(ctx, ns) + _ = waitNamespaceDelete(ctx, ns) } for _, rs := range resources { - DeleteCRD(ctx, rs) + deleteCRD(ctx, rs) } } func installVeleroForAPIGroups(ctx context.Context) error { - if err := EnsureClusterExists(ctx); err != nil { + if err := ensureClusterExists(ctx); err != nil { return errors.Wrap(err, "check cluster exists") } // Pass global variables to option parameters. - options, err := GetProviderVeleroInstallOptions( + options, err := getProviderVeleroInstallOptions( cloudProvider, cloudCredentialsFile, bslBucket, @@ -441,14 +458,14 @@ func installVeleroForAPIGroups(ctx context.Context) error { options.Features = "EnableAPIGroupVersions" options.Image = veleroImage - if err := InstallVeleroServer(options); err != nil { + if err := installVeleroServer(options); err != nil { return errors.WithMessagef(err, "Failed to install Velero in the cluster") } return nil } -func InstallCRD(ctx context.Context, crd resourceCRD) error { +func installCRD(ctx context.Context, crd resourceCRD) error { fmt.Printf("Install CRD %s.\n", crd.url) cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", crd.url) @@ -458,17 +475,17 @@ func InstallCRD(ctx context.Context, crd resourceCRD) error { } fmt.Println("Wait for CRD to be ready.") - if err := WaitForPodContainers(ctx, crd.ns); err != nil { + if err := waitForPodContainers(ctx, crd.ns); err != nil { return err } return err } -// WaitForPodContainers will get the pods and container status in a namespace. +// waitForPodContainers will get the pods and container status in a namespace. // If the ratio of the number of containers running to total in a pod is not 1, // it is not ready. Otherwise, if all container ratios are 1, the pod is running. -func WaitForPodContainers(ctx context.Context, ns string) error { +func waitForPodContainers(ctx context.Context, ns string) error { err := wait.Poll(3*time.Second, 4*time.Minute, func() (bool, error) { cmd := exec.CommandContext(ctx, "kubectl", "get", "pods", "-n", ns) stdout, stderr, err := veleroexec.RunCommand(cmd) @@ -498,7 +515,7 @@ func WaitForPodContainers(ctx context.Context, ns string) error { return err } -func DeleteCRD(ctx context.Context, crd resourceCRD) error { +func deleteCRD(ctx context.Context, crd resourceCRD) error { fmt.Println("Delete CRD", crd.url) cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crd.url, "--wait") @@ -529,7 +546,7 @@ func DeleteCRD(ctx context.Context, crd resourceCRD) error { return err } -func RestartPods(ctx context.Context, ns string) error { +func restartPods(ctx context.Context, ns string) error { fmt.Printf("Restart pods in %s namespace.\n", ns) cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns) @@ -537,7 +554,7 @@ func RestartPods(ctx context.Context, ns string) error { if err == nil { fmt.Println("Wait for pods to be ready.") - if err := WaitForPodContainers(ctx, ns); err != nil { + if err := waitForPodContainers(ctx, ns); err != nil { return err } } @@ -545,7 +562,7 @@ func RestartPods(ctx context.Context, ns string) error { return err } -func InstallCR(ctx context.Context, crFile, ns string) error { +func installCR(ctx context.Context, crFile, ns string) error { retries := 5 var stderr string var err error @@ -565,7 +582,7 @@ func InstallCR(ctx context.Context, crFile, ns string) error { return errors.Wrap(err, stderr) } -func WaitNamespaceDelete(ctx context.Context, ns string) error { +func waitNamespaceDelete(ctx context.Context, ns string) error { err := wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { cmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", ns) diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index b7b423a1d2..bbc8e27097 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -77,14 +93,14 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel } // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, +func RunKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, useVolumeSnapshots bool) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) timeout := 10 * time.Minute interval := 5 * time.Second - if err := CreateNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { + if err := createNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } @@ -103,8 +119,8 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac return errors.Wrap(err, "Failed to generate data") } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { - VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) + if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { + veleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } @@ -113,13 +129,13 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac return errors.Wrap(err, "Failed to simulate a disaster") } // wait for ns delete - err := WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) + err := waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) if err != nil { return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) } - if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - VeleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) + if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + veleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } @@ -140,13 +156,13 @@ func RunKibishiiTests(client TestClient, providerName, veleroCLI, veleroNamespac return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace) } // wait for ns delete - if err = WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace); err != nil { + if err = waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) } fmt.Printf("kibishii test completed successfully\n") return nil } -func waitForKibishiiPods(ctx context.Context, client TestClient, kibishiiNamespace string) error { - return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) +func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error { + return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index 02c26bceb7..951d002e3c 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -36,7 +52,7 @@ func getProviderPlugins(providerName string) []string { } // GetProviderVeleroInstallOptions returns Velero InstallOptions for the provider. -func GetProviderVeleroInstallOptions( +func getProviderVeleroInstallOptions( pluginProvider, credentialsFile, objectStoreBucket, @@ -76,14 +92,14 @@ func GetProviderVeleroInstallOptions( return io, nil } -// InstallVeleroServer installs velero in the cluster. -func InstallVeleroServer(io *cliinstall.InstallOptions) error { +// installVeleroServer installs velero in the cluster. +func installVeleroServer(io *cliinstall.InstallOptions) error { vo, err := io.AsVeleroOptions() if err != nil { return errors.Wrap(err, "Failed to translate InstallOptions to VeleroOptions for Velero") } - client, err := NewTestClient() + client, err := newTestClient() if err != nil { return errors.Wrap(err, "Failed to instantiate cluster client for installing Velero") } @@ -112,8 +128,8 @@ func InstallVeleroServer(io *cliinstall.InstallOptions) error { return nil } -// CheckBackupPhase uses veleroCLI to inspect the phase of a Velero backup. -func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, +// checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup. +func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, expectedPhase velerov1api.BackupPhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json", backupName) @@ -156,8 +172,8 @@ func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str return nil } -// CheckRestorePhase uses veleroCLI to inspect the phase of a Velero restore. -func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, +// checkRestorePhase uses veleroCLI to inspect the phase of a Velero restore. +func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, expectedPhase velerov1api.RestorePhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json", restoreName) @@ -200,8 +216,8 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st return nil } -// VeleroBackupNamespace uses the veleroCLI to backup a namespace. -func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, +// veleroBackupNamespace uses the veleroCLI to backup a namespace. +func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, useVolumeSnapshots bool) error { args := []string{ "--namespace", veleroNamespace, @@ -227,13 +243,13 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespac if err != nil { return err } - err = CheckBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) + err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) return err } -// VeleroRestore uses the veleroCLI to restore from a Velero backup. -func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { +// veleroRestore uses the veleroCLI to restore from a Velero backup. +func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName, "--from-backup", backupName, "--wait") @@ -244,10 +260,10 @@ func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string if err != nil { return err } - return CheckRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) + return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) } -func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, +func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, features string) error { @@ -261,11 +277,11 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider } } - err := EnsureClusterExists(ctx) + err := ensureClusterExists(ctx) if err != nil { return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } - veleroInstallOptions, err := GetProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, + veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, getProviderPlugins(objectStoreProvider), features) if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) @@ -273,7 +289,7 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri veleroInstallOptions.UseRestic = !useVolumeSnapshots veleroInstallOptions.Image = veleroImage veleroInstallOptions.Namespace = veleroNamespace - err = InstallVeleroServer(veleroInstallOptions) + err = installVeleroServer(veleroInstallOptions) if err != nil { return errors.WithMessagef(err, "Failed to install Velero in the cluster") } @@ -281,7 +297,7 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri return nil } -func VeleroUninstall(client kbclient.Client, installVelero bool, veleroNamespace string) error { +func veleroUninstall(client kbclient.Client, installVelero bool, veleroNamespace string) error { if installVelero { err := uninstall.Run(client, veleroNamespace) if err != nil { @@ -291,7 +307,7 @@ func VeleroUninstall(client kbclient.Client, installVelero bool, veleroNamespace return nil } -func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { +func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName) describeCmd.Stdout = os.Stdout describeCmd.Stderr = os.Stderr @@ -309,7 +325,7 @@ func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace str return nil } -func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { +func veleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "describe", restoreName) describeCmd.Stdout = os.Stdout describeCmd.Stderr = os.Stderr @@ -327,7 +343,7 @@ func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace st return nil } -func VeleroCreateBackupLocation(ctx context.Context, +func veleroCreateBackupLocation(ctx context.Context, veleroCLI string, veleroNamespace string, name string, @@ -364,9 +380,9 @@ func VeleroCreateBackupLocation(ctx context.Context, return bslCreateCmd.Run() } -// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and +// veleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { +func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { for _, plugin := range getProviderPlugins(provider) { stdoutBuf := new(bytes.Buffer) stderrBuf := new(bytes.Buffer) From cf5c53ccd10567d53b34c91b87d999bfc24612aa Mon Sep 17 00:00:00 2001 From: Carlisia Date: Mon, 22 Mar 2021 09:07:25 -0700 Subject: [PATCH 4/5] More cleanup Signed-off-by: Carlisia --- test/e2e/client.go | 3 ++- test/e2e/enable_api_group_versions_test.go | 21 ++++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/test/e2e/client.go b/test/e2e/client.go index 2ab5d54906..591d6c576f 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -17,9 +17,10 @@ limitations under the License. package e2e import ( - "github.com/vmware-tanzu/velero/pkg/client" "k8s.io/client-go/kubernetes" kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/client" ) type testClient struct { diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index d9e6bbeb99..79a8f96eb8 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -56,15 +56,12 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", group = "music.example.io" certManagerCRD = map[string]resourceCRD{ - cert: resourceCRD{ + cert: { url: "testdata/enable_api_group_versions/cert-manager.yaml", ns: "cert-manager", }, } - err = installCRD(ctx, certManagerCRD[cert]) - Expect(err).NotTo(HaveOccurred()) - uuidgen, err = uuid.NewRandom() Expect(err).NotTo(HaveOccurred()) }) @@ -81,7 +78,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", Context("When EnableAPIGroupVersions flag is set", func() { It("Should back up API group version and restore by version priority", func() { - Expect(RunEnableAPIGroupVersionsTests( + Expect(runEnableAPIGroupVersionsTests( ctx, client, resource, @@ -92,7 +89,7 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", }) }) -func RunEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string, certManager resourceCRD) error { +func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string, certManager resourceCRD) error { const src = "source" const tgt = "target" const srcCRs = "srcCRs" @@ -223,8 +220,14 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso for i, tc := range tests { fmt.Printf("\n====== Test Case %d ======\n", i) - err := installCRD(ctx, tc.resources[src]) + err := installCRD(ctx, certManager) + Expect(err).NotTo(HaveOccurred()) + + err = installCRD(ctx, tc.resources[src]) if err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + }) return errors.Wrap(err, "installing music-system CRD for source cluster") } @@ -291,6 +294,10 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso } if err := waitNamespaceDelete(ctx, ns); err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } } From daddf28366ffeb197b5a1cd29c0f5a943704beca Mon Sep 17 00:00:00 2001 From: Carlisia Date: Tue, 23 Mar 2021 10:16:36 -0700 Subject: [PATCH 5/5] Add documentation Signed-off-by: Carlisia --- test/e2e/README.md | 6 +++++ test/e2e/backup_test.go | 2 +- test/e2e/client.go | 29 +++++++++++++++++----- test/e2e/common.go | 8 +++--- test/e2e/enable_api_group_versions_test.go | 9 ++++--- test/e2e/kibishii_tests.go | 4 +-- test/e2e/velero_utils.go | 6 ++--- 7 files changed, 44 insertions(+), 20 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 013fb5cb3a..93c7e68782 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -108,5 +108,11 @@ The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` make variable. Th ## Adding tests +### API clients +When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients. + +### Test cleanup +If your test creates resources, be sure it is removing these resources everywhere where there is a potential failure point so even if the test ends with a failure, the environemnt will be cleaned up for the next test. + ### Tips Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index ead5cacea7..3d370ba4eb 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -68,7 +68,7 @@ func backup_restore_test(useVolumeSnapshots bool) { }) AfterEach(func() { - err = veleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) + err = veleroUninstall(client.kubebuilder, installVelero, veleroNamespace) Expect(err).To(Succeed()) }) diff --git a/test/e2e/client.go b/test/e2e/client.go index 591d6c576f..0b19939342 100644 --- a/test/e2e/client.go +++ b/test/e2e/client.go @@ -23,12 +23,29 @@ import ( "github.com/vmware-tanzu/velero/pkg/client" ) +// testClient contains different API clients that are in use throughout +// the e2e tests. + type testClient struct { - Kubebuilder kbclient.Client - ClientGo kubernetes.Interface - DynamicFactory client.DynamicFactory + kubebuilder kbclient.Client + + // clientGo returns a client-go API client. + // + // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the + // controller runtime framework by v2.0, it is the intent to remove all + // client-go API clients. Please use the controller runtime to make API calls for tests. + clientGo kubernetes.Interface + + // dynamicFactory returns a client-go API client for retrieving dynamic clients + // for GroupVersionResources and GroupVersionKinds. + // + // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the + // controller runtime framework by v2.0, it is the intent to remove all + // client-go API clients. Please use the controller runtime to make API calls for tests. + dynamicFactory client.DynamicFactory } +// newTestClient returns a set of ready-to-use API clients. func newTestClient() (testClient, error) { config, err := client.LoadConfig() if err != nil { @@ -55,8 +72,8 @@ func newTestClient() (testClient, error) { factory := client.NewDynamicFactory(dynamicClient) return testClient{ - Kubebuilder: kb, - ClientGo: clientGo, - DynamicFactory: factory, + kubebuilder: kb, + clientGo: clientGo, + dynamicFactory: factory, }, nil } diff --git a/test/e2e/common.go b/test/e2e/common.go index c6632335a4..ce553d8157 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -41,7 +41,7 @@ func ensureClusterExists(ctx context.Context) error { // createNamespace creates a kubernetes namespace func createNamespace(ctx context.Context, client testClient, namespace string) error { ns := builder.ForNamespace(namespace).Result() - _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -51,7 +51,7 @@ func createNamespace(ctx context.Context, client testClient, namespace string) e // waitForNamespaceDeletion waits for namespace to be deleted. func waitForNamespaceDeletion(interval, timeout time.Duration, client testClient, ns string) error { err := wait.PollImmediate(interval, timeout, func() (bool, error) { - _, err := client.ClientGo.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) + _, err := client.clientGo.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil @@ -77,7 +77,7 @@ func createSecretFromFiles(ctx context.Context, client testClient, namespace str } secret := builder.ForSecret(namespace, name).Data(data).Result() - _, err := client.ClientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) return err } @@ -87,7 +87,7 @@ func waitForPods(ctx context.Context, client testClient, namespace string, pods interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { for _, podName := range pods { - checkPod, err := client.ClientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + checkPod, err := client.clientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) if err != nil { return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning)) } diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 79a8f96eb8..d12c11e98a 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -285,7 +285,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso } for _, ns := range tc.namespaces { - if err := client.ClientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { testCleanup(ctx, client, nil, []resourceCRD{ certManager, tc.resources[src], @@ -313,7 +313,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso // Apply config map if there is one. if tc.cm != nil { - _, err := client.ClientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { testCleanup(ctx, client, tc.namespaces, []resourceCRD{ certManager, @@ -415,7 +415,7 @@ func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, reso tc.resources[tgt], }) - err = veleroUninstall(client.Kubebuilder, installVelero, veleroNamespace) + err = veleroUninstall(client.kubebuilder, installVelero, veleroNamespace) if err != nil { return err } @@ -429,10 +429,11 @@ type resourceCRD struct { } func testCleanup(ctx context.Context, client testClient, namespaces []string, resources []resourceCRD) { + fmt.Println("Running test cleanup for [APIGroup]...") // Delete namespaces created for CRs for _, ns := range namespaces { fmt.Println("Delete namespace", ns) - _ = client.ClientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + _ = client.clientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) _ = waitNamespaceDelete(ctx, ns) } diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index bbc8e27097..ade425ebf8 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -125,7 +125,7 @@ func RunKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac } fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := client.ClientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, "Failed to simulate a disaster") } // wait for ns delete @@ -152,7 +152,7 @@ func RunKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespac return errors.Wrap(err, "Failed to verify data generated by kibishii") } - if err := client.ClientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace) } // wait for ns delete diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index 951d002e3c..e5bfe9da72 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -106,19 +106,19 @@ func installVeleroServer(io *cliinstall.InstallOptions) error { errorMsg := "\n\nError installing Velero. Use `kubectl logs deploy/velero -n velero` to check the deploy logs" resources := install.AllResources(vo) - err = install.Install(client.DynamicFactory, resources, os.Stdout) + err = install.Install(client.dynamicFactory, resources, os.Stdout) if err != nil { return errors.Wrap(err, errorMsg) } fmt.Println("Waiting for Velero deployment to be ready.") - if _, err = install.DeploymentIsReady(client.DynamicFactory, io.Namespace); err != nil { + if _, err = install.DeploymentIsReady(client.dynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } if io.UseRestic { fmt.Println("Waiting for Velero restic daemonset to be ready.") - if _, err = install.DaemonSetIsReady(client.DynamicFactory, io.Namespace); err != nil { + if _, err = install.DaemonSetIsReady(client.dynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } }