From 1116307bb370007a7d9fcaf0828e135420f4cc7c Mon Sep 17 00:00:00 2001 From: Ming Date: Fri, 10 Dec 2021 17:03:14 +0800 Subject: [PATCH] Upgrade e2e test vsphere plugin Signed-off-by: Ming --- changelogs/unreleased/4440-mqiu | 1 + test/e2e/util/k8s/configmap.go | 28 +++++++++++ test/e2e/util/k8s/secret.go | 51 ++++++++++++++++++++ test/e2e/util/velero/install.go | 72 ++++++++++++++++++++++++++-- test/e2e/util/velero/velero_utils.go | 10 +++- 5 files changed, 156 insertions(+), 6 deletions(-) create mode 100644 changelogs/unreleased/4440-mqiu diff --git a/changelogs/unreleased/4440-mqiu b/changelogs/unreleased/4440-mqiu new file mode 100644 index 00000000000..0d8a196bddf --- /dev/null +++ b/changelogs/unreleased/4440-mqiu @@ -0,0 +1 @@ +Upgrade e2e test vsphere plugin diff --git a/test/e2e/util/k8s/configmap.go b/test/e2e/util/k8s/configmap.go index 73bb1911c72..91fcf97cdd1 100644 --- a/test/e2e/util/k8s/configmap.go +++ b/test/e2e/util/k8s/configmap.go @@ -17,10 +17,17 @@ limitations under the License. package k8s import ( + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" "golang.org/x/net/context" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + waitutil "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" ) @@ -45,3 +52,24 @@ func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) e return true, nil }) } + +func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, error) { + return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) +} + +func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error { + if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete configmap in namespace %q", ns)) + } + return waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + logrus.Debugf("configmap %q in namespace %q is still being deleted...", name, ns) + return false, nil + }) +} diff --git a/test/e2e/util/k8s/secret.go b/test/e2e/util/k8s/secret.go index 9701559eb50..3c3682c933f 100644 --- a/test/e2e/util/k8s/secret.go +++ b/test/e2e/util/k8s/secret.go @@ -17,10 +17,18 @@ limitations under the License. package k8s import ( + "fmt" + "time" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/net/context" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" + waitutil "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" ) @@ -34,6 +42,23 @@ func CreateSecret(c clientset.Interface, ns, name string, labels map[string]stri return c.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{}) } +func WaitForSecretDelete(c clientset.Interface, ns, name string) error { + if err := c.CoreV1().Secrets(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete secret in namespace %q", ns)) + } + return waitutil.PollImmediateInfinite(5*time.Second, + func() (bool, error) { + if _, err := c.CoreV1().Secrets(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + logrus.Debugf("secret %q in namespace %q is still being deleted...", name, ns) + return false, nil + }) +} + // WaitForSecretsComplete uses c to wait for completions to complete for the Job jobName in namespace ns. func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error { return wait.Poll(PollInterval, PollTimeout, func() (bool, error) { @@ -44,3 +69,29 @@ func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error return true, nil }) } + +func GetSecret(c clientset.Interface, ns, secretName string) (*v1.Secret, error) { + return c.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) +} + +//CreateVCCredentialSecret refer to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md +func CreateVCCredentialSecret(c clientset.Interface, veleroNamespace string) error { + secret, err := GetSecret(c, "kube-system", "vsphere-config-secret") + if err != nil { + return err + } + vsphereCfg, exist := secret.Data["csi-vsphere.conf"] + if !exist { + return errors.New("failed to retrieve csi-vsphere config") + } + se := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "velero-vsphere-config-secret", + Namespace: veleroNamespace, + }, + Type: v1.SecretTypeOpaque, + Data: map[string][]byte{"csi-vsphere.conf": vsphereCfg}, + } + _, err = c.CoreV1().Secrets(veleroNamespace).Create(context.TODO(), se, metav1.CreateOptions{}) + return err +} diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go index b843fb0be34..7f110e9b368 100644 --- a/test/e2e/util/velero/install.go +++ b/test/e2e/util/velero/install.go @@ -29,9 +29,11 @@ import ( "github.com/pkg/errors" apps "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" velerexec "github.com/vmware-tanzu/velero/pkg/util/exec" @@ -62,16 +64,20 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin if err != nil { return errors.WithMessage(err, "Failed to get provider plugins") } + err = EnsureClusterExists(ctx) + if err != nil { + return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") + } + // TODO - handle this better if veleroCfg.CloudProvider == "vsphere" { // We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified veleroCfg.ObjectStoreProvider = "aws" - } - err = EnsureClusterExists(ctx) - if err != nil { - return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists") + if err := configvSpherePlugin(); err != nil { + return errors.WithMessagef(err, "Failed to config vsphere plugin") + } } veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, @@ -97,6 +103,64 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin return nil } +//configvSpherePlugin refers to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md +func configvSpherePlugin() error { + cli, err := NewTestClient() + if err != nil { + return errors.WithMessagef(err, "Failed to instantiate cluster client to config vsphere plugin") + } + vsphereSecret := "velero-vsphere-config-secret" + configmaptName := "velero-vsphere-plugin-config" + if err := clearupvSpherePluginConfig(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret, configmaptName); err != nil { + return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", VeleroCfg.VeleroNamespace) + } + if err := CreateNamespace(context.Background(), cli, VeleroCfg.VeleroNamespace); err != nil { + return errors.WithMessagef(err, "Failed to create Velero %s namespace", VeleroCfg.VeleroNamespace) + } + if err := CreateVCCredentialSecret(cli.ClientGo, VeleroCfg.VeleroNamespace); err != nil { + return errors.WithMessagef(err, "Failed to create virtual center credential secret in %s namespace", VeleroCfg.VeleroNamespace) + } + if err := WaitForSecretsComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret); err != nil { + return errors.Wrap(err, "Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system") + } + _, err = CreateConfigMap(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName, map[string]string{ + "cluster_flavor": "VANILLA", + "vsphere_secret_name": vsphereSecret, + "vsphere_secret_namespace": VeleroCfg.VeleroNamespace, + }) + if err != nil { + return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", VeleroCfg.VeleroNamespace) + } + err = WaitForConfigMapComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, VeleroCfg.VeleroNamespace)) + } + return nil +} + +func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error { + //clear secret + _, err := GetSecret(c, ns, secretName) + if err == nil { //exist + if err := WaitForSecretDelete(c, ns, secretName); err != nil { + return errors.WithMessagef(err, "Failed to clear up vsphere plugin secret in %s namespace", ns) + } + } else if !apierrors.IsNotFound(err) { + return errors.WithMessagef(err, "Failed to retrieve vsphere plugin secret in %s namespace", ns) + } + + //clear configmap + _, err = GetConfigmap(c, ns, configMapName) + if err == nil { + if err := WaitForConfigmapDelete(c, ns, configMapName); err != nil { + return errors.WithMessagef(err, "Failed to clear up vsphere plugin configmap in %s namespace", ns) + } + } else if !apierrors.IsNotFound(err) { + return errors.WithMessagef(err, "Failed to retrieve vsphere plugin configmap in %s namespace", ns) + } + return nil +} + func installVeleroServer(ctx context.Context, cli string, options *installOptions) error { args := []string{"install"} namespace := "velero" diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go index d32c9f577f7..c1fe583485b 100644 --- a/test/e2e/util/velero/velero_utils.go +++ b/test/e2e/util/velero/velero_utils.go @@ -63,13 +63,19 @@ var pluginsMatrix = map[string]map[string][]string{ "v1.7": { "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, - "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, + }, + "v1.8": { + "aws": {"velero/velero-plugin-for-aws:v1.3.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.3.0"}, + "vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, "gcp": {"velero/velero-plugin-for-gcp:v1.3.0"}, }, "main": { "aws": {"velero/velero-plugin-for-aws:main"}, "azure": {"velero/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"}, + "vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"}, "gcp": {"velero/velero-plugin-for-gcp:main"}, }, }