Merge pull request #4440 from qiuming-best/upgrade-e2e-vsphere-plugin

Upgrade e2e test vsphere plugin
vmware-tanzu · Dec 16, 2021 · e350ce5 · e350ce5
2 parents 5aaeb3e + d8b1ed7
commit e350ce5
Show file tree

Hide file tree

Showing 5 changed files with 150 additions and 6 deletions.
diff --git a/changelogs/unreleased/4440-mqiu b/changelogs/unreleased/4440-mqiu
@@ -0,0 +1 @@
+Upgrade e2e test vsphere plugin
diff --git a/test/e2e/util/k8s/configmap.go b/test/e2e/util/k8s/configmap.go
@@ -17,10 +17,17 @@ limitations under the License.
 package k8s
 
 import (
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	waitutil "k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 )
 
@@ -45,3 +52,24 @@ func WaitForConfigMapComplete(c clientset.Interface, ns, configmapName string) e
 		return true, nil
 	})
 }
+
+func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, error) {
+	return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{})
+}
+
+func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error {
+	if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to delete  configmap in namespace %q", ns))
+	}
+	return waitutil.PollImmediateInfinite(5*time.Second,
+		func() (bool, error) {
+			if _, err := c.CoreV1().ConfigMaps(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil {
+				if apierrors.IsNotFound(err) {
+					return true, nil
+				}
+				return false, err
+			}
+			logrus.Debugf("configmap %q in namespace %q is still being deleted...", name, ns)
+			return false, nil
+		})
+}
diff --git a/test/e2e/util/k8s/secret.go b/test/e2e/util/k8s/secret.go
@@ -17,10 +17,18 @@ limitations under the License.
 package k8s
 
 import (
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
 	"golang.org/x/net/context"
 	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	waitutil "k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 )
 
@@ -34,6 +42,23 @@ func CreateSecret(c clientset.Interface, ns, name string, labels map[string]stri
 	return c.CoreV1().Secrets(ns).Create(context.TODO(), secret, metav1.CreateOptions{})
 }
 
+func WaitForSecretDelete(c clientset.Interface, ns, name string) error {
+	if err := c.CoreV1().Secrets(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to delete  secret in namespace %q", ns))
+	}
+	return waitutil.PollImmediateInfinite(5*time.Second,
+		func() (bool, error) {
+			if _, err := c.CoreV1().Secrets(ns).Get(context.TODO(), ns, metav1.GetOptions{}); err != nil {
+				if apierrors.IsNotFound(err) {
+					return true, nil
+				}
+				return false, err
+			}
+			logrus.Debugf("secret %q in namespace %q is still being deleted...", name, ns)
+			return false, nil
+		})
+}
+
 // WaitForSecretsComplete uses c to wait for completions to complete for the Job jobName in namespace ns.
 func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error {
 	return wait.Poll(PollInterval, PollTimeout, func() (bool, error) {
@@ -44,3 +69,29 @@ func WaitForSecretsComplete(c clientset.Interface, ns, secretName string) error
 		return true, nil
 	})
 }
+
+func GetSecret(c clientset.Interface, ns, secretName string) (*v1.Secret, error) {
+	return c.CoreV1().Secrets(ns).Get(context.TODO(), secretName, metav1.GetOptions{})
+}
+
+//CreateVCCredentialSecret refer to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
+func CreateVCCredentialSecret(c clientset.Interface, veleroNamespace string) error {
+	secret, err := GetSecret(c, "kube-system", "vsphere-config-secret")
+	if err != nil {
+		return err
+	}
+	vsphereCfg, exist := secret.Data["csi-vsphere.conf"]
+	if !exist {
+		return errors.New("failed to retrieve csi-vsphere config")
+	}
+	se := &v1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "velero-vsphere-config-secret",
+			Namespace: veleroNamespace,
+		},
+		Type: v1.SecretTypeOpaque,
+		Data: map[string][]byte{"csi-vsphere.conf": vsphereCfg},
+	}
+	_, err = c.CoreV1().Secrets(veleroNamespace).Create(context.TODO(), se, metav1.CreateOptions{})
+	return err
+}
diff --git a/test/e2e/util/velero/install.go b/test/e2e/util/velero/install.go
@@ -29,9 +29,11 @@ import (
 	"github.com/pkg/errors"
 	apps "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
 
 	"github.com/vmware-tanzu/velero/pkg/cmd/cli/install"
 	velerexec "github.com/vmware-tanzu/velero/pkg/util/exec"
@@ -62,16 +64,20 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin
 	if err != nil {
 		return errors.WithMessage(err, "Failed to get provider plugins")
 	}
+	err = EnsureClusterExists(ctx)
+	if err != nil {
+		return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
+	}
+
 	// TODO - handle this better
 	if veleroCfg.CloudProvider == "vsphere" {
 		// We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the
 		// backup, but needed to pick up the provider plugins earlier.  vSphere plugin no longer needs a Volume
 		// Snapshot location specified
 		veleroCfg.ObjectStoreProvider = "aws"
-	}
-	err = EnsureClusterExists(ctx)
-	if err != nil {
-		return errors.WithMessage(err, "Failed to ensure Kubernetes cluster exists")
+		if err := configvSpherePlugin(); err != nil {
+			return errors.WithMessagef(err, "Failed to config vsphere plugin")
+		}
 	}
 
 	veleroInstallOptions, err := getProviderVeleroInstallOptions(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,
@@ -96,6 +102,64 @@ func VeleroInstall(ctx context.Context, veleroCfg *VerleroConfig, features strin
 	return nil
 }
 
+//configvSpherePlugin refers to https://github.com/vmware-tanzu/velero-plugin-for-vsphere/blob/v1.3.0/docs/vanilla.md
+func configvSpherePlugin() error {
+	cli, err := NewTestClient()
+	if err != nil {
+		return errors.WithMessagef(err, "Failed to instantiate cluster client to config vsphere plugin")
+	}
+	vsphereSecret := "velero-vsphere-config-secret"
+	configmaptName := "velero-vsphere-plugin-config"
+	if err := clearupvSpherePluginConfig(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret, configmaptName); err != nil {
+		return errors.WithMessagef(err, "Failed to clear up vsphere plugin config %s namespace", VeleroCfg.VeleroNamespace)
+	}
+	if err := CreateNamespace(context.Background(), cli, VeleroCfg.VeleroNamespace); err != nil {
+		return errors.WithMessagef(err, "Failed to create Velero %s namespace", VeleroCfg.VeleroNamespace)
+	}
+	if err := CreateVCCredentialSecret(cli.ClientGo, VeleroCfg.VeleroNamespace); err != nil {
+		return errors.WithMessagef(err, "Failed to create virtual center credential secret in %s namespace", VeleroCfg.VeleroNamespace)
+	}
+	if err := WaitForSecretsComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, vsphereSecret); err != nil {
+		return errors.Wrap(err, "Failed to ensure velero-vsphere-config-secret secret completion in namespace kube-system")
+	}
+	_, err = CreateConfigMap(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName, map[string]string{
+		"cluster_flavor":           "VANILLA",
+		"vsphere_secret_name":      vsphereSecret,
+		"vsphere_secret_namespace": VeleroCfg.VeleroNamespace,
+	})
+	if err != nil {
+		return errors.WithMessagef(err, "Failed to create velero-vsphere-plugin-config configmap in %s namespace", VeleroCfg.VeleroNamespace)
+	}
+	err = WaitForConfigMapComplete(cli.ClientGo, VeleroCfg.VeleroNamespace, configmaptName)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("Failed to ensure configmap %s completion in namespace: %s", configmaptName, VeleroCfg.VeleroNamespace))
+	}
+	return nil
+}
+
+func clearupvSpherePluginConfig(c clientset.Interface, ns, secretName, configMapName string) error {
+	//clear secret
+	_, err := GetSecret(c, ns, secretName)
+	if err == nil { //exist
+		if err := WaitForSecretDelete(c, ns, secretName); err != nil {
+			return errors.WithMessagef(err, "Failed to clear up vsphere plugin secret in %s namespace", ns)
+		}
+	} else if !apierrors.IsNotFound(err) {
+		return errors.WithMessagef(err, "Failed to retrieve vsphere plugin secret in %s namespace", ns)
+	}
+
+	//clear configmap
+	_, err = GetConfigmap(c, ns, configMapName)
+	if err == nil {
+		if err := WaitForConfigmapDelete(c, ns, configMapName); err != nil {
+			return errors.WithMessagef(err, "Failed to clear up vsphere plugin configmap in %s namespace", ns)
+		}
+	} else if !apierrors.IsNotFound(err) {
+		return errors.WithMessagef(err, "Failed to retrieve vsphere plugin configmap in %s namespace", ns)
+	}
+	return nil
+}
+
 func installVeleroServer(ctx context.Context, cli string, options *installOptions) error {
 	args := []string{"install"}
 	namespace := "velero"

diff --git a/test/e2e/util/velero/velero_utils.go b/test/e2e/util/velero/velero_utils.go
@@ -63,13 +63,13 @@ var pluginsMatrix = map[string]map[string][]string{
 	"v1.7": {
 		"aws":     {"velero/velero-plugin-for-aws:v1.3.0"},
 		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
-		"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"vsphere": {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
 		"gcp":     {"velero/velero-plugin-for-gcp:v1.3.0"},
 	},
 	"main": {
 		"aws":     {"velero/velero-plugin-for-aws:main"},
 		"azure":   {"velero/velero-plugin-for-microsoft-azure:main"},
-		"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
+		"vsphere": {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
 		"gcp":     {"velero/velero-plugin-for-gcp:main"},
 	},
 }