diff --git a/pkg/client/factory.go b/pkg/client/factory.go index 6ecc63be74..5d55ccedd6 100644 --- a/pkg/client/factory.go +++ b/pkg/client/factory.go @@ -23,9 +23,11 @@ import ( "github.com/pkg/errors" "github.com/spf13/pflag" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + k8scheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" @@ -45,7 +47,8 @@ type Factory interface { // DynamicClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. DynamicClient() (dynamic.Interface, error) - // KubebuilderClient returns a Kubernetes dynamic client. It uses the following priority to specify the cluster + // KubebuilderClient returns a client for the controller runtime framework. It adds Kubernetes and Velero + // types to its scheme. It uses the following priority to specify the cluster // configuration: --kubeconfig flag, KUBECONFIG environment variable, in-cluster configuration. KubebuilderClient() (kbclient.Client, error) // SetBasename changes the basename for an already-constructed client. @@ -151,6 +154,8 @@ func (f *factory) KubebuilderClient() (kbclient.Client, error) { scheme := runtime.NewScheme() velerov1api.AddToScheme(scheme) + k8scheme.AddToScheme(scheme) + apiextv1beta1.AddToScheme(scheme) kubebuilderClient, err := kbclient.New(clientConfig, kbclient.Options{ Scheme: scheme, }) diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index eb82cbbfd8..d2ed61c6af 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -255,10 +255,7 @@ func (o *InstallOptions) Run(c *cobra.Command, f client.Factory) error { return err } - resources, err = install.AllResources(vo) - if err != nil { - return err - } + resources = install.AllResources(vo) } if _, err := output.PrintWithFormat(c, resources); err != nil { diff --git a/pkg/cmd/cli/uninstall/uninstall.go b/pkg/cmd/cli/uninstall/uninstall.go index f7ce824cad..910a8deddf 100644 --- a/pkg/cmd/cli/uninstall/uninstall.go +++ b/pkg/cmd/cli/uninstall/uninstall.go @@ -19,57 +19,126 @@ package uninstall import ( "context" "fmt" - "strings" "time" "github.com/pkg/errors" "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + + // apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + kubeerrs "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" - - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" "github.com/vmware-tanzu/velero/pkg/client" "github.com/vmware-tanzu/velero/pkg/cmd" + "github.com/vmware-tanzu/velero/pkg/cmd/cli" "github.com/vmware-tanzu/velero/pkg/cmd/util/output" "github.com/vmware-tanzu/velero/pkg/install" - "github.com/vmware-tanzu/velero/pkg/util/kube" ) -// Uninstall uninstalls all components deployed using velero install command -func Uninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclientset.Clientset, veleroNamespace string) error { - if veleroNamespace == "" { - veleroNamespace = "velero" +// NewCommand creates a cobra command. +func NewCommand(f client.Factory) *cobra.Command { + c := &cobra.Command{ + Use: "uninstall", + Short: "Uninstall Velero", + Long: `Uninstall Velero along with the CRDs. + +The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero). + `, + Example: `# velero uninstall -n backup`, + Run: func(c *cobra.Command, args []string) { + + if !cli.GetConfirmation() { + // Don't do anything unless we get confirmation + return + } + + kbClient, err := f.KubebuilderClient() + cmd.CheckError(err) + cmd.CheckError(Run(kbClient, f.Namespace())) + }, } - err := DeleteNamespace(ctx, client, veleroNamespace) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing Velero namespace %s", veleroNamespace) + + output.BindFlags(c.Flags()) + output.ClearOutputFlagDefault(c) + return c +} + +// Run removes all components that were deployed using the Velero install command +func Run(kbClient kbclient.Client, namespace string) error { + var errs []error + + // namespace + ns := &corev1.Namespace{} + key := kbclient.ObjectKey{Name: namespace} + if err := kbClient.Get(context.Background(), key, ns); err != nil { + if apierrors.IsNotFound(err) { + fmt.Printf("Velero installation namespace %q does not exist, skipping.\n", namespace) + } else { + errs = append(errs, errors.WithStack(err)) + } + } else { + if ns.Status.Phase == corev1.NamespaceTerminating { + fmt.Printf("Velero installation namespace %q is terminating.\n", namespace) + } else { + if err := kbClient.Delete(context.Background(), ns); err != nil { + errs = append(errs, errors.WithStack(err)) + } + } } - rolebinding := install.ClusterRoleBinding(veleroNamespace) + time.Sleep(time.Second * 60) - err = client.RbacV1().ClusterRoleBindings().Delete(ctx, rolebinding.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing Velero cluster role binding %s", rolebinding) + // rolebinding + crb := install.ClusterRoleBinding(namespace) + key = kbclient.ObjectKey{Name: crb.Name, Namespace: namespace} + if err := kbClient.Get(context.Background(), key, crb); err != nil { + if apierrors.IsNotFound(err) { + fmt.Printf("Velero installation rolebinding %q does not exist, skipping.\n", crb.Name) + } else { + errs = append(errs, errors.WithStack(err)) + } + } else { + if err := kbClient.Delete(context.Background(), crb); err != nil { + errs = append(errs, errors.WithStack(err)) + } } - veleroLabels := labels.FormatLabels(install.Labels()) - crds, err := extensionsClient.ApiextensionsV1().CustomResourceDefinitions().List(ctx, metav1.ListOptions{ - LabelSelector: veleroLabels, - }) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed listing Velero crds") + // CRDs + veleroLabels := labels.FormatLabels(install.Labels()) + crdList := apiextv1beta1.CustomResourceDefinitionList{} + opts := kbclient.ListOptions{ + Namespace: namespace, + Raw: &metav1.ListOptions{ + LabelSelector: veleroLabels, + }, } - for _, removeCRD := range crds.Items { - err = extensionsClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, removeCRD.ObjectMeta.Name, metav1.DeleteOptions{}) - if err != nil { - return errors.WithMessagef(err, "Uninstall failed removing CRD %s", removeCRD.ObjectMeta.Name) + if err := kbClient.List(context.Background(), &crdList, &opts); err != nil { + errs = append(errs, errors.WithStack(err)) + } else { + if len(crdList.Items) == 0 { + fmt.Print("Velero CRDs do not exist, skipping.\n") + } else { + for _, crd := range crdList.Items { + if err := kbClient.Delete(context.Background(), &crd); err != nil { + errs = append(errs, errors.WithStack(err)) + } + } } } - fmt.Println("Uninstalled Velero") + if kubeerrs.NewAggregate(errs) != nil { + fmt.Printf("Errors while attempting to uninstall Velero: %q", kubeerrs.NewAggregate(errs)) + return kubeerrs.NewAggregate(errs) + } + + fmt.Println("Velero uninstalled ⛵") return nil } @@ -89,26 +158,3 @@ func DeleteNamespace(ctx context.Context, client *kubernetes.Clientset, namespac return false, nil }) } - -// NewCommand creates a cobra command. -func NewCommand(f client.Factory) *cobra.Command { - c := &cobra.Command{ - Use: "uninstall", - Short: "Uninstall Velero", - Long: `Uninstall Velero along with the CRDs. - -The '--namespace' flag can be used to specify the namespace where velero is installed (default: velero). - `, - Example: `# velero uninstall -n backup`, - Run: func(c *cobra.Command, args []string) { - veleroNs := strings.TrimSpace(f.Namespace()) - cl, extCl, err := kube.GetClusterClient() - cmd.CheckError(err) - cmd.CheckError(Uninstall(context.Background(), cl, extCl, veleroNs)) - }, - } - - output.BindFlags(c.Flags()) - output.ClearOutputFlagDefault(c) - return c -} diff --git a/pkg/install/install.go b/pkg/install/install.go index 88e12a71a0..7d1871ed51 100644 --- a/pkg/install/install.go +++ b/pkg/install/install.go @@ -234,16 +234,19 @@ func createResource(r *unstructured.Unstructured, factory client.DynamicFactory, format := strings.Join([]string{id, ": ", f, "\n"}, "") fmt.Fprintf(w, format, a...) } - log("attempting to create resource") c, err := CreateClient(r, factory, w) if err != nil { return err } - if _, err := c.Create(r); apierrors.IsAlreadyExists(err) { - log("already exists, proceeding") - } else if err != nil { + log("attempting to create resource") + _, err = c.Create(r) + if err != nil { + if apierrors.IsAlreadyExists(err) { + log("already exists, proceeding") + return nil + } return errors.Wrapf(err, "Error creating resource %s", id) } diff --git a/pkg/install/resources.go b/pkg/install/resources.go index 885232045f..9ca6393881 100644 --- a/pkg/install/resources.go +++ b/pkg/install/resources.go @@ -245,7 +245,7 @@ func AllCRDs() *unstructured.UnstructuredList { // AllResources returns a list of all resources necessary to install Velero, in the appropriate order, into a Kubernetes cluster. // Items are unstructured, since there are different data types returned. -func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) { +func AllResources(o *VeleroOptions) *unstructured.UnstructuredList { resources := AllCRDs() ns := Namespace(o.Namespace) @@ -317,5 +317,5 @@ func AllResources(o *VeleroOptions) (*unstructured.UnstructuredList, error) { appendUnstructured(resources, ds) } - return resources, nil + return resources } diff --git a/pkg/util/kube/utils.go b/pkg/util/kube/utils.go index c8262080d3..994e8f370c 100644 --- a/pkg/util/kube/utils.go +++ b/pkg/util/kube/utils.go @@ -24,15 +24,12 @@ import ( "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" corev1listers "k8s.io/client-go/listers/core/v1" - "k8s.io/client-go/tools/clientcmd" ) // NamespaceAndName returns a string in the format / @@ -220,26 +217,3 @@ func IsUnstructuredCRDReady(crd *unstructured.Unstructured) (bool, error) { return (isEstablished && namesAccepted), nil } - -// GetClusterClient instantiates and returns a client for the cluster. -func GetClusterClient() (*kubernetes.Clientset, *apiextensionsclientset.Clientset, error) { - loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() - configOverrides := &clientcmd.ConfigOverrides{} - kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, configOverrides) - clientConfig, err := kubeConfig.ClientConfig() - if err != nil { - return nil, nil, errors.WithStack(err) - } - - client, err := kubernetes.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - extensionClientSet, err := apiextensionsclientset.NewForConfig(clientConfig) - if err != nil { - return nil, nil, errors.WithStack(err) - } - - return client, extensionClientSet, nil -} diff --git a/test/e2e/README.md b/test/e2e/README.md index 08eeb40c32..93c7e68782 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -105,3 +105,14 @@ For example, E2E tests can be run from Velero repository roots using the command Velero E2E tests uses [Ginkgo](https://onsi.github.io/ginkgo/) testing framework which allows a subset of the tests to be run using the [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) flags to ginkgo. The `-focus` flag is passed to ginkgo using the `GINKGO_FOCUS` make variable. This can be used to focus on specific tests. + +## Adding tests + +### API clients +When adding a test, aim to instantiate an API client only once at the beginning of the test. There is a constructor `newTestClient` that facilitates the configuration and instantiation of clients. Also, please use the `kubebuilder` runtime controller client for any new test, as we will phase out usage of `client-go` API clients. + +### Test cleanup +If your test creates resources, be sure it is removing these resources everywhere where there is a potential failure point so even if the test ends with a failure, the environemnt will be cleaned up for the next test. + +### Tips +Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. diff --git a/test/e2e/backup_test.go b/test/e2e/backup_test.go index 20dc96ecb9..3d370ba4eb 100644 --- a/test/e2e/backup_test.go +++ b/test/e2e/backup_test.go @@ -1,18 +1,29 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( "context" "flag" "fmt" - "time" "github.com/google/uuid" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - "k8s.io/client-go/kubernetes" - - "github.com/vmware-tanzu/velero/pkg/util/kube" ) var ( @@ -34,12 +45,13 @@ func backup_restore_with_restic() { func backup_restore_test(useVolumeSnapshots bool) { var ( - client *kubernetes.Clientset - extensionsClient *apiextensionsclientset.Clientset - backupName string - restoreName string + backupName string + restoreName string ) + client, err := newTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for backup tests") + BeforeEach(func() { if useVolumeSnapshots && cloudProvider == "kind" { Skip("Volume snapshots not supported on kind") @@ -49,21 +61,15 @@ func backup_restore_test(useVolumeSnapshots bool) { uuidgen, err = uuid.NewRandom() Expect(err).To(Succeed()) if installVelero { - Expect(VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, + Expect(veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, useVolumeSnapshots, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "")).To(Succeed()) } - client, extensionsClient, err = kube.GetClusterClient() - Expect(err).To(Succeed(), "Failed to instantiate cluster client") }) AfterEach(func() { - if installVelero { - timeoutCTX, _ := context.WithTimeout(context.Background(), time.Minute) - err := VeleroUninstall(timeoutCTX, client, extensionsClient, veleroNamespace) - Expect(err).To(Succeed()) - } - + err = veleroUninstall(client.kubebuilder, installVelero, veleroNamespace) + Expect(err).To(Succeed()) }) When("kibishii is the sample workload", func() { @@ -89,7 +95,7 @@ func backup_restore_test(useVolumeSnapshots bool) { Skip("no additional BSL credentials given, not running multiple BackupStorageLocation with unique credentials tests") } - Expect(VeleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) + Expect(veleroAddPluginsForProvider(context.TODO(), veleroCLI, veleroNamespace, additionalBSLProvider)).To(Succeed()) // Create Secret for additional BSL secretName := fmt.Sprintf("bsl-credentials-%s", uuidgen) @@ -98,11 +104,11 @@ func backup_restore_test(useVolumeSnapshots bool) { secretKey: additionalBSLCredentials, } - Expect(CreateSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) + Expect(createSecretFromFiles(context.TODO(), client, veleroNamespace, secretName, files)).To(Succeed()) // Create additional BSL using credential additionalBsl := fmt.Sprintf("bsl-%s", uuidgen) - Expect(VeleroCreateBackupLocation(context.TODO(), + Expect(veleroCreateBackupLocation(context.TODO(), veleroCLI, veleroNamespace, additionalBsl, diff --git a/test/e2e/client.go b/test/e2e/client.go new file mode 100644 index 0000000000..0b19939342 --- /dev/null +++ b/test/e2e/client.go @@ -0,0 +1,79 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "k8s.io/client-go/kubernetes" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/client" +) + +// testClient contains different API clients that are in use throughout +// the e2e tests. + +type testClient struct { + kubebuilder kbclient.Client + + // clientGo returns a client-go API client. + // + // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the + // controller runtime framework by v2.0, it is the intent to remove all + // client-go API clients. Please use the controller runtime to make API calls for tests. + clientGo kubernetes.Interface + + // dynamicFactory returns a client-go API client for retrieving dynamic clients + // for GroupVersionResources and GroupVersionKinds. + // + // Deprecated, TODO(2.0): presuming all controllers and resources are converted to the + // controller runtime framework by v2.0, it is the intent to remove all + // client-go API clients. Please use the controller runtime to make API calls for tests. + dynamicFactory client.DynamicFactory +} + +// newTestClient returns a set of ready-to-use API clients. +func newTestClient() (testClient, error) { + config, err := client.LoadConfig() + if err != nil { + return testClient{}, err + } + + f := client.NewFactory("e2e", config) + + clientGo, err := f.KubeClient() + if err != nil { + return testClient{}, err + } + + kb, err := f.KubebuilderClient() + if err != nil { + return testClient{}, err + } + + dynamicClient, err := f.DynamicClient() + if err != nil { + return testClient{}, err + } + + factory := client.NewDynamicFactory(dynamicClient) + + return testClient{ + kubebuilder: kb, + clientGo: clientGo, + dynamicFactory: factory, + }, nil +} diff --git a/test/e2e/common.go b/test/e2e/common.go index 6dbfc02914..ce553d8157 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -13,30 +29,29 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/builder" ) -// EnsureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. -func EnsureClusterExists(ctx context.Context) error { +// ensureClusterExists returns whether or not a kubernetes cluster exists for tests to be run on. +func ensureClusterExists(ctx context.Context) error { return exec.CommandContext(ctx, "kubectl", "cluster-info").Run() } -// CreateNamespace creates a kubernetes namespace -func CreateNamespace(ctx context.Context, client *kubernetes.Clientset, namespace string) error { +// createNamespace creates a kubernetes namespace +func createNamespace(ctx context.Context, client testClient, namespace string) error { ns := builder.ForNamespace(namespace).Result() - _, err := client.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } return err } -// WaitForNamespaceDeletion Waits for namespace to be deleted. -func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernetes.Clientset, ns string) error { +// waitForNamespaceDeletion waits for namespace to be deleted. +func waitForNamespaceDeletion(interval, timeout time.Duration, client testClient, ns string) error { err := wait.PollImmediate(interval, timeout, func() (bool, error) { - _, err := client.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) + _, err := client.clientGo.CoreV1().Namespaces().Get(context.TODO(), ns, metav1.GetOptions{}) if err != nil { if apierrors.IsNotFound(err) { return true, nil @@ -49,7 +64,7 @@ func WaitForNamespaceDeletion(interval, timeout time.Duration, client *kubernete return err } -func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, namespace string, name string, files map[string]string) error { +func createSecretFromFiles(ctx context.Context, client testClient, namespace string, name string, files map[string]string) error { data := make(map[string][]byte) for key, filePath := range files { @@ -62,19 +77,17 @@ func CreateSecretFromFiles(ctx context.Context, client *kubernetes.Clientset, na } secret := builder.ForSecret(namespace, name).Data(data).Result() - _, err := client.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().Secrets(namespace).Create(ctx, secret, metav1.CreateOptions{}) return err } -/* - Waits until all of the pods have gone to PodRunning state -*/ -func WaitForPods(ctx context.Context, client *kubernetes.Clientset, namespace string, pods []string) error { +// waitForPods waits until all of the pods have gone to PodRunning state +func waitForPods(ctx context.Context, client testClient, namespace string, pods []string) error { timeout := 10 * time.Minute interval := 5 * time.Second err := wait.PollImmediate(interval, timeout, func() (bool, error) { for _, podName := range pods { - checkPod, err := client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + checkPod, err := client.clientGo.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) if err != nil { return false, errors.WithMessage(err, fmt.Sprintf("Failed to verify pod %s/%s is %s", namespace, podName, corev1api.PodRunning)) } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index ecac96bd4c..0429070ec3 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( diff --git a/test/e2e/enable_api_group_versions_test.go b/test/e2e/enable_api_group_versions_test.go index 20be75bf6a..d12c11e98a 100644 --- a/test/e2e/enable_api_group_versions_test.go +++ b/test/e2e/enable_api_group_versions_test.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -12,17 +28,12 @@ import ( "github.com/google/uuid" - "github.com/vmware-tanzu/velero/pkg/util/kube" - - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/pkg/errors" corev1api "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/builder" veleroexec "github.com/vmware-tanzu/velero/pkg/util/exec" @@ -30,27 +41,26 @@ import ( var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", func() { var ( - resource, group string - certMgrCRD map[string]string - client *kubernetes.Clientset - extensionsClient *apiextensionsclient.Clientset - err error - ctx = context.Background() + resource, group string + certManagerCRD map[string]resourceCRD + err error + ctx = context.Background() ) + const cert = "cert" + + client, err := newTestClient() + Expect(err).To(Succeed(), "Failed to instantiate cluster client for group version tests") BeforeEach(func() { resource = "rockbands" group = "music.example.io" - certMgrCRD = map[string]string{ - "url": "testdata/enable_api_group_versions/cert-manager.yaml", - "namespace": "cert-manager", - } - - client, extensionsClient, err = kube.GetClusterClient() // Currently we ignore the API extensions client - Expect(err).NotTo(HaveOccurred()) - - err = InstallCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) - Expect(err).NotTo(HaveOccurred()) + certManagerCRD = + map[string]resourceCRD{ + cert: { + url: "testdata/enable_api_group_versions/cert-manager.yaml", + ns: "cert-manager", + }, + } uuidgen, err = uuid.NewRandom() Expect(err).NotTo(HaveOccurred()) @@ -63,30 +73,32 @@ var _ = Describe("[APIGroup] Velero tests with various CRD API group versions", cmd = exec.CommandContext(ctx, "kubectl", "delete", "crd", "rockbands.music.example.io") _, _, _ = veleroexec.RunCommand(cmd) - _ = DeleteCRD(ctx, certMgrCRD["url"], certMgrCRD["namespace"]) + _ = deleteCRD(ctx, certManagerCRD[cert]) }) Context("When EnableAPIGroupVersions flag is set", func() { It("Should back up API group version and restore by version priority", func() { - Expect(RunEnableAPIGroupVersionsTests( + Expect(runEnableAPIGroupVersionsTests( ctx, + client, resource, group, - client, - extensionsClient, + certManagerCRD[cert], )).To(Succeed(), "Failed to successfully backup and restore multiple API Groups") }) }) }) -func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, client *kubernetes.Clientset, - extensionsClient *apiextensionsclient.Clientset) error { +func runEnableAPIGroupVersionsTests(ctx context.Context, client testClient, resource, group string, certManager resourceCRD) error { + const src = "source" + const tgt = "target" + const srcCRs = "srcCRs" + tests := []struct { name string namespaces []string - srcCRD map[string]string + resources map[string]resourceCRD srcCRs map[string]string - tgtCRD map[string]string tgtVer string cm *corev1api.ConfigMap gvs map[string][]string @@ -94,18 +106,20 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, }{ { name: "Target and source cluster preferred versions match; Preferred version v1 is restored (Priority 1, Case A).", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-a-source.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: { + url: "testdata/enable_api_group_versions/case-a-source.yaml", + ns: "music-system", + }, + tgt: { + url: "testdata/enable_api_group_versions/case-a-target.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", "v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-a-target.yaml", - "namespace": "music-system", - }, tgtVer: "v1", cm: nil, want: map[string]map[string]string{ @@ -120,19 +134,21 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, }, { name: "Latest common non-preferred supported version v2beta2 is restored (Priority 3, Case D).", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: { + url: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", + ns: "music-system", + }, + tgt: { + url: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml", "v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml", "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", - "namespace": "music-system", - }, tgtVer: "v2beta2", cm: nil, want: map[string]map[string]string{ @@ -148,37 +164,41 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, }, { name: "No common supported versions means no rockbands custom resource is restored.", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-a-source.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: { + url: "testdata/enable_api_group_versions/case-a-source.yaml", + ns: "music-system", + }, + tgt: { + url: "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", "v1alpha1": "testdata/enable_api_group_versions/music_v1alpha1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-b-target-manually-added-mutations.yaml", - "namespace": "music-system", - }, tgtVer: "", cm: nil, want: nil, }, { name: "User config map overrides Priority 3, Case D and restores v2beta1", - srcCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", - "namespace": "music-system", + resources: map[string]resourceCRD{ + src: { + url: "testdata/enable_api_group_versions/case-b-source-manually-added-mutations.yaml", + ns: "music-system", + }, + tgt: { + url: "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", + ns: "music-system", + }, }, srcCRs: map[string]string{ "v2beta2": "testdata/enable_api_group_versions/music_v2beta2_rockband.yaml", "v2beta1": "testdata/enable_api_group_versions/music_v2beta1_rockband.yaml", "v1": "testdata/enable_api_group_versions/music_v1_rockband.yaml", }, - tgtCRD: map[string]string{ - "url": "testdata/enable_api_group_versions/case-d-target-manually-added-mutations.yaml", - "namespace": "music-system", - }, tgtVer: "v2beta1", cm: builder.ForConfigMap(veleroNamespace, "enableapigroupversions").Data( "restoreResourcesVersionPriority", @@ -200,29 +220,43 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, for i, tc := range tests { fmt.Printf("\n====== Test Case %d ======\n", i) - err := InstallCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]) + err := installCRD(ctx, certManager) + Expect(err).NotTo(HaveOccurred()) + + err = installCRD(ctx, tc.resources[src]) if err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + }) return errors.Wrap(err, "installing music-system CRD for source cluster") } for version, cr := range tc.srcCRs { ns := resource + "-src-" + version + tc.namespaces = append(tc.namespaces, ns) - if err := CreateNamespace(ctx, client, ns); err != nil { + if err := createNamespace(ctx, client, ns); err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "creating %s namespace", ns) } - if err := InstallCR(ctx, cr, ns); err != nil { + if err := installCR(ctx, cr, ns); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "installing %s custom resource on source cluster namespace %s", cr, ns) } - tc.namespaces = append(tc.namespaces, ns) } // TODO - Velero needs to be installed AFTER CRDs are installed because of https://github.com/vmware-tanzu/velero/issues/3471 // Once that issue is fixed, we should install Velero once for the test suite if installVelero { - VeleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, + veleroInstall(context.Background(), veleroImage, veleroNamespace, cloudProvider, objectStoreProvider, false, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, "EnableAPIGroupVersions" /* TODO - remove this when the feature flag is removed */) fmt.Println("Sleep 20s to wait for Velero to stabilize after install.") @@ -232,42 +266,71 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, backup := "backup-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) namespacesStr := strings.Join(tc.namespaces, ",") - err = VeleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) + err = veleroBackupNamespace(ctx, veleroCLI, veleroNamespace, backup, namespacesStr, "", false) if err != nil { - VeleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) + veleroBackupLogs(ctx, veleroCLI, veleroNamespace, backup) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "backing up %s namespaces on source cluster", namespacesStr) } // Delete music-system CRD and controllers installed on source cluster. - if err := DeleteCRD(ctx, tc.srcCRD["url"], tc.srcCRD["namespace"]); err != nil { + if err := deleteCRD(ctx, tc.resources[src]); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + }) return errors.Wrapf(err, "deleting music-system CRD from source cluster") } for _, ns := range tc.namespaces { - if err := client.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}); err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } - if err := WaitNamespaceDelete(ctx, ns); err != nil { + if err := waitNamespaceDelete(ctx, ns); err != nil { + testCleanup(ctx, client, nil, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "deleting %s namespace from source cluster", ns) } } // Install music-system CRD for target cluster. - if err := InstallCRD(ctx, tc.tgtCRD["url"], tc.tgtCRD["namespace"]); err != nil { + if err := installCRD(ctx, tc.resources[tgt]); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + }) return errors.Wrapf(err, "installing music-system CRD for target cluster") } // Apply config map if there is one. if tc.cm != nil { - _, err := client.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) + _, err := client.clientGo.CoreV1().ConfigMaps(veleroNamespace).Create(ctx, tc.cm, metav1.CreateOptions{}) if err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrap(err, "creating config map with user version priorities") } } // Reset Velero to recognize music-system CRD. - if err := RestartPods(ctx, veleroNamespace); err != nil { + if err := restartPods(ctx, veleroNamespace); err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrapf(err, "restarting Velero pods") } fmt.Println("Sleep 20s to wait for Velero to stabilize after restart.") @@ -277,13 +340,22 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, restore := "restore-rockbands-" + uuidgen.String() + "-" + strconv.Itoa(i) if tc.want != nil { - if err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { - VeleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) + if err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup); err != nil { + veleroRestoreLogs(ctx, veleroCLI, veleroNamespace, restore) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrapf(err, "restoring %s namespaces on target cluster", namespacesStr) } annoSpec, err := resourceInfo(ctx, group, tc.tgtVer, resource) if err != nil { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.Wrapf( err, "get annotation and spec from %s.%s/%s object", @@ -300,6 +372,11 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, annoSpec["annotations"], tc.want["annotations"], ) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.New(msg) } @@ -310,58 +387,68 @@ func RunEnableAPIGroupVersionsTests(ctx context.Context, resource, group string, annoSpec["specs"], tc.want["specs"], ) + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.New(msg) } } else { // No custom resource should have been restored. Expect "no resource found" // error during restore. - err := VeleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) + err := veleroRestore(ctx, veleroCLI, veleroNamespace, restore, backup) if err.Error() != "Unexpected restore phase got PartiallyFailed, expecting Completed" { + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) return errors.New("expected error but not none") } } - // Delete namespaces created for CRs - for _, ns := range tc.namespaces { - fmt.Println("Delete namespace", ns) - _ = client.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) - _ = WaitNamespaceDelete(ctx, ns) - } + testCleanup(ctx, client, tc.namespaces, []resourceCRD{ + certManager, + tc.resources[src], + tc.resources[tgt], + }) - // Delete source cluster music-system CRD - _ = DeleteCRD( - ctx, - tc.srcCRD["url"], - tc.srcCRD["namespace"], - ) - - // Delete target cluster music-system CRD - _ = DeleteCRD( - ctx, - tc.tgtCRD["url"], - tc.srcCRD["namespace"], - ) - - // Uninstall Velero - if installVelero { - err = VeleroUninstall(ctx, client, extensionsClient, veleroNamespace) - if err != nil { - return err - } + err = veleroUninstall(client.kubebuilder, installVelero, veleroNamespace) + if err != nil { + return err } } return nil } +type resourceCRD struct { + url, ns string +} + +func testCleanup(ctx context.Context, client testClient, namespaces []string, resources []resourceCRD) { + fmt.Println("Running test cleanup for [APIGroup]...") + // Delete namespaces created for CRs + for _, ns := range namespaces { + fmt.Println("Delete namespace", ns) + _ = client.clientGo.CoreV1().Namespaces().Delete(ctx, ns, metav1.DeleteOptions{}) + _ = waitNamespaceDelete(ctx, ns) + } + + for _, rs := range resources { + deleteCRD(ctx, rs) + } +} + func installVeleroForAPIGroups(ctx context.Context) error { - if err := EnsureClusterExists(ctx); err != nil { + if err := ensureClusterExists(ctx); err != nil { return errors.Wrap(err, "check cluster exists") } // Pass global variables to option parameters. - options, err := GetProviderVeleroInstallOptions( + options, err := getProviderVeleroInstallOptions( cloudProvider, cloudCredentialsFile, bslBucket, @@ -379,34 +466,34 @@ func installVeleroForAPIGroups(ctx context.Context) error { options.Features = "EnableAPIGroupVersions" options.Image = veleroImage - if err := InstallVeleroServer(options); err != nil { - return errors.Wrap(err, "install velero server") + if err := installVeleroServer(options); err != nil { + return errors.WithMessagef(err, "Failed to install Velero in the cluster") } return nil } -func InstallCRD(ctx context.Context, crdFile, ns string) error { - fmt.Printf("Install CRD %s.\n", crdFile) +func installCRD(ctx context.Context, crd resourceCRD) error { + fmt.Printf("Install CRD %s.\n", crd.url) - cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", crdFile) + cmd := exec.CommandContext(ctx, "kubectl", "apply", "-f", crd.url) _, stderr, err := veleroexec.RunCommand(cmd) if err != nil { return errors.Wrap(err, stderr) } fmt.Println("Wait for CRD to be ready.") - if err := WaitForPodContainers(ctx, ns); err != nil { + if err := waitForPodContainers(ctx, crd.ns); err != nil { return err } return err } -// WaitForPodContainers will get the pods and container status in a namespace. +// waitForPodContainers will get the pods and container status in a namespace. // If the ratio of the number of containers running to total in a pod is not 1, // it is not ready. Otherwise, if all container ratios are 1, the pod is running. -func WaitForPodContainers(ctx context.Context, ns string) error { +func waitForPodContainers(ctx context.Context, ns string) error { err := wait.Poll(3*time.Second, 4*time.Minute, func() (bool, error) { cmd := exec.CommandContext(ctx, "kubectl", "get", "pods", "-n", ns) stdout, stderr, err := veleroexec.RunCommand(cmd) @@ -436,9 +523,9 @@ func WaitForPodContainers(ctx context.Context, ns string) error { return err } -func DeleteCRD(ctx context.Context, crdFile, ns string) error { - fmt.Println("Delete CRD", crdFile) - cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crdFile, "--wait") +func deleteCRD(ctx context.Context, crd resourceCRD) error { + fmt.Println("Delete CRD", crd.url) + cmd := exec.CommandContext(ctx, "kubectl", "delete", "-f", crd.url, "--wait") _, stderr, err := veleroexec.RunCommand(cmd) if strings.Contains(stderr, "not found") { @@ -450,7 +537,7 @@ func DeleteCRD(ctx context.Context, crdFile, ns string) error { } err = wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { - cmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", ns) + cmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", crd.ns) stdout, stderr, err := veleroexec.RunCommand(cmd) if strings.Contains(stderr, "not found") { @@ -461,13 +548,13 @@ func DeleteCRD(ctx context.Context, crdFile, ns string) error { return false, errors.Wrap(err, stderr) } - re := regexp.MustCompile(ns) + re := regexp.MustCompile(crd.ns) return re.MatchString(stdout), nil }) return err } -func RestartPods(ctx context.Context, ns string) error { +func restartPods(ctx context.Context, ns string) error { fmt.Printf("Restart pods in %s namespace.\n", ns) cmd := exec.CommandContext(ctx, "kubectl", "delete", "pod", "--all", "-n", ns) @@ -475,7 +562,7 @@ func RestartPods(ctx context.Context, ns string) error { if err == nil { fmt.Println("Wait for pods to be ready.") - if err := WaitForPodContainers(ctx, ns); err != nil { + if err := waitForPodContainers(ctx, ns); err != nil { return err } } @@ -483,7 +570,7 @@ func RestartPods(ctx context.Context, ns string) error { return err } -func InstallCR(ctx context.Context, crFile, ns string) error { +func installCR(ctx context.Context, crFile, ns string) error { retries := 5 var stderr string var err error @@ -503,7 +590,7 @@ func InstallCR(ctx context.Context, crFile, ns string) error { return errors.Wrap(err, stderr) } -func WaitNamespaceDelete(ctx context.Context, ns string) error { +func waitNamespaceDelete(ctx context.Context, ns string) error { err := wait.Poll(1*time.Second, 3*time.Minute, func() (bool, error) { cmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", ns) diff --git a/test/e2e/kibishii_tests.go b/test/e2e/kibishii_tests.go index 624be7ad01..ade425ebf8 100644 --- a/test/e2e/kibishii_tests.go +++ b/test/e2e/kibishii_tests.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -8,7 +24,6 @@ import ( "github.com/pkg/errors" "golang.org/x/net/context" - "k8s.io/client-go/kubernetes" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -78,14 +93,14 @@ func verifyData(ctx context.Context, namespace string, levels int, filesPerLevel } // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, +func RunKibishiiTests(client testClient, providerName, veleroCLI, veleroNamespace, backupName, restoreName, backupLocation string, useVolumeSnapshots bool) error { fiveMinTimeout, _ := context.WithTimeout(context.Background(), 5*time.Minute) oneHourTimeout, _ := context.WithTimeout(context.Background(), time.Minute*60) timeout := 10 * time.Minute interval := 5 * time.Second - if err := CreateNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { + if err := createNamespace(fiveMinTimeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } @@ -104,23 +119,23 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrap(err, "Failed to generate data") } - if err := VeleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { - VeleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) + if err := veleroBackupNamespace(oneHourTimeout, veleroCLI, veleroNamespace, backupName, kibishiiNamespace, backupLocation, useVolumeSnapshots); err != nil { + veleroBackupLogs(fiveMinTimeout, veleroCLI, veleroNamespace, backupName) return errors.Wrapf(err, "Failed to backup kibishii namespace %s", kibishiiNamespace) } fmt.Printf("Simulating a disaster by removing namespace %s\n", kibishiiNamespace) - if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, "Failed to simulate a disaster") } // wait for ns delete - err := WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) + err := waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace) if err != nil { return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) } - if err := VeleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { - VeleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) + if err := veleroRestore(oneHourTimeout, veleroCLI, veleroNamespace, restoreName, backupName); err != nil { + veleroRestoreLogs(fiveMinTimeout, veleroCLI, veleroNamespace, restoreName) return errors.Wrapf(err, "Restore %s failed from backup %s", restoreName, backupName) } @@ -137,17 +152,17 @@ func RunKibishiiTests(client *kubernetes.Clientset, providerName, veleroCLI, vel return errors.Wrap(err, "Failed to verify data generated by kibishii") } - if err := client.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { + if err := client.clientGo.CoreV1().Namespaces().Delete(oneHourTimeout, kibishiiNamespace, metav1.DeleteOptions{}); err != nil { return errors.Wrapf(err, "Failed to cleanup %s wrokload namespace", kibishiiNamespace) } // wait for ns delete - if err = WaitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace); err != nil { + if err = waitForNamespaceDeletion(interval, timeout, client, kibishiiNamespace); err != nil { return errors.Wrapf(err, fmt.Sprintf("Failed to wait for deletion of namespace %s", kibishiiNamespace)) } fmt.Printf("kibishii test completed successfully\n") return nil } -func waitForKibishiiPods(ctx context.Context, client *kubernetes.Clientset, kibishiiNamespace string) error { - return WaitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) +func waitForKibishiiPods(ctx context.Context, client testClient, kibishiiNamespace string) error { + return waitForPods(ctx, client, kibishiiNamespace, []string{"jump-pad", "etcd0", "etcd1", "etcd2", "kibishii-deployment-0", "kibishii-deployment-1"}) } diff --git a/test/e2e/velero_utils.go b/test/e2e/velero_utils.go index d23d969e21..e5bfe9da72 100644 --- a/test/e2e/velero_utils.go +++ b/test/e2e/velero_utils.go @@ -1,3 +1,19 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package e2e import ( @@ -12,12 +28,9 @@ import ( "strings" "github.com/pkg/errors" - "k8s.io/client-go/kubernetes" - - apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + kbclient "sigs.k8s.io/controller-runtime/pkg/client" velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" - "github.com/vmware-tanzu/velero/pkg/client" cliinstall "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" "github.com/vmware-tanzu/velero/pkg/cmd/cli/uninstall" "github.com/vmware-tanzu/velero/pkg/cmd/util/flag" @@ -39,7 +52,7 @@ func getProviderPlugins(providerName string) []string { } // GetProviderVeleroInstallOptions returns Velero InstallOptions for the provider. -func GetProviderVeleroInstallOptions( +func getProviderVeleroInstallOptions( pluginProvider, credentialsFile, objectStoreBucket, @@ -79,52 +92,44 @@ func GetProviderVeleroInstallOptions( return io, nil } -// InstallVeleroServer installs velero in the cluster. -func InstallVeleroServer(io *cliinstall.InstallOptions) error { - config, err := client.LoadConfig() - if err != nil { - return err - } - +// installVeleroServer installs velero in the cluster. +func installVeleroServer(io *cliinstall.InstallOptions) error { vo, err := io.AsVeleroOptions() if err != nil { return errors.Wrap(err, "Failed to translate InstallOptions to VeleroOptions for Velero") } - f := client.NewFactory("e2e", config) - resources, err := install.AllResources(vo) + client, err := newTestClient() if err != nil { - return errors.Wrap(err, "Failed to install Velero in the cluster") + return errors.Wrap(err, "Failed to instantiate cluster client for installing Velero") } - dynamicClient, err := f.DynamicClient() - if err != nil { - return err - } - factory := client.NewDynamicFactory(dynamicClient) errorMsg := "\n\nError installing Velero. Use `kubectl logs deploy/velero -n velero` to check the deploy logs" - err = install.Install(factory, resources, os.Stdout) + resources := install.AllResources(vo) + err = install.Install(client.dynamicFactory, resources, os.Stdout) if err != nil { return errors.Wrap(err, errorMsg) } fmt.Println("Waiting for Velero deployment to be ready.") - if _, err = install.DeploymentIsReady(factory, io.Namespace); err != nil { + if _, err = install.DeploymentIsReady(client.dynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } if io.UseRestic { fmt.Println("Waiting for Velero restic daemonset to be ready.") - if _, err = install.DaemonSetIsReady(factory, io.Namespace); err != nil { + if _, err = install.DaemonSetIsReady(client.dynamicFactory, io.Namespace); err != nil { return errors.Wrap(err, errorMsg) } } + fmt.Printf("Velero is installed and ready to be tested in the %s namespace! ⛵ \n", io.Namespace) + return nil } -// CheckBackupPhase uses veleroCLI to inspect the phase of a Velero backup. -func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, +// checkBackupPhase uses veleroCLI to inspect the phase of a Velero backup. +func checkBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, expectedPhase velerov1api.BackupPhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "get", "-o", "json", backupName) @@ -167,8 +172,8 @@ func CheckBackupPhase(ctx context.Context, veleroCLI string, veleroNamespace str return nil } -// CheckRestorePhase uses veleroCLI to inspect the phase of a Velero restore. -func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, +// checkRestorePhase uses veleroCLI to inspect the phase of a Velero restore. +func checkRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, expectedPhase velerov1api.RestorePhase) error { checkCMD := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "get", "-o", "json", restoreName) @@ -211,8 +216,8 @@ func CheckRestorePhase(ctx context.Context, veleroCLI string, veleroNamespace st return nil } -// VeleroBackupNamespace uses the veleroCLI to backup a namespace. -func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, +// veleroBackupNamespace uses the veleroCLI to backup a namespace. +func veleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string, namespace string, backupLocation string, useVolumeSnapshots bool) error { args := []string{ "--namespace", veleroNamespace, @@ -238,13 +243,13 @@ func VeleroBackupNamespace(ctx context.Context, veleroCLI string, veleroNamespac if err != nil { return err } - err = CheckBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) + err = checkBackupPhase(ctx, veleroCLI, veleroNamespace, backupName, velerov1api.BackupPhaseCompleted) return err } -// VeleroRestore uses the veleroCLI to restore from a Velero backup. -func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { +// veleroRestore uses the veleroCLI to restore from a Velero backup. +func veleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string, backupName string) error { restoreCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "create", "restore", restoreName, "--from-backup", backupName, "--wait") @@ -255,10 +260,10 @@ func VeleroRestore(ctx context.Context, veleroCLI string, veleroNamespace string if err != nil { return err } - return CheckRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) + return checkRestorePhase(ctx, veleroCLI, veleroNamespace, restoreName, velerov1api.RestorePhaseCompleted) } -func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, +func veleroInstall(ctx context.Context, veleroImage string, veleroNamespace string, cloudProvider string, objectStoreProvider string, useVolumeSnapshots bool, cloudCredentialsFile string, bslBucket string, bslPrefix string, bslConfig string, vslConfig string, features string) error { @@ -272,11 +277,11 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider } } - err := EnsureClusterExists(ctx) + err := ensureClusterExists(ctx) if err != nil { - return errors.WithMessage(err, "Failed to ensure kubernetes cluster exists") + return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") } - veleroInstallOptions, err := GetProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, + veleroInstallOptions, err := getProviderVeleroInstallOptions(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, vslConfig, getProviderPlugins(objectStoreProvider), features) if err != nil { return errors.WithMessagef(err, "Failed to get Velero InstallOptions for plugin provider %s", objectStoreProvider) @@ -284,18 +289,25 @@ func VeleroInstall(ctx context.Context, veleroImage string, veleroNamespace stri veleroInstallOptions.UseRestic = !useVolumeSnapshots veleroInstallOptions.Image = veleroImage veleroInstallOptions.Namespace = veleroNamespace - err = InstallVeleroServer(veleroInstallOptions) + err = installVeleroServer(veleroInstallOptions) if err != nil { - return errors.WithMessagef(err, "Failed to install Velero in cluster") + return errors.WithMessagef(err, "Failed to install Velero in the cluster") } + return nil } -func VeleroUninstall(ctx context.Context, client *kubernetes.Clientset, extensionsClient *apiextensionsclient.Clientset, veleroNamespace string) error { - return uninstall.Uninstall(ctx, client, extensionsClient, veleroNamespace) +func veleroUninstall(client kbclient.Client, installVelero bool, veleroNamespace string) error { + if installVelero { + err := uninstall.Run(client, veleroNamespace) + if err != nil { + return err + } + } + return nil } -func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { +func veleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace string, backupName string) error { describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "backup", "describe", backupName) describeCmd.Stdout = os.Stdout describeCmd.Stderr = os.Stderr @@ -313,7 +325,7 @@ func VeleroBackupLogs(ctx context.Context, veleroCLI string, veleroNamespace str return nil } -func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { +func veleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace string, restoreName string) error { describeCmd := exec.CommandContext(ctx, veleroCLI, "--namespace", veleroNamespace, "restore", "describe", restoreName) describeCmd.Stdout = os.Stdout describeCmd.Stderr = os.Stderr @@ -331,7 +343,7 @@ func VeleroRestoreLogs(ctx context.Context, veleroCLI string, veleroNamespace st return nil } -func VeleroCreateBackupLocation(ctx context.Context, +func veleroCreateBackupLocation(ctx context.Context, veleroCLI string, veleroNamespace string, name string, @@ -368,9 +380,9 @@ func VeleroCreateBackupLocation(ctx context.Context, return bslCreateCmd.Run() } -// VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and +// veleroAddPluginsForProvider determines which plugins need to be installed for a provider and // installs them in the current Velero installation, skipping over those that are already installed. -func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { +func veleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string) error { for _, plugin := range getProviderPlugins(provider) { stdoutBuf := new(bytes.Buffer) stderrBuf := new(bytes.Buffer)