Virtualization E2E tests on OpenStack (#1511)

* Avoid deleting previously installed CirrOS image. Only clean it up if the test downloaded it in the first place. Signed-off-by: Matthew Arnold <[email protected]> * Default OpenStack provider to no KVM emulation. Allow KVM emulation to be configured, and add an OpenStack provider that copies AWS except for leaving emulation turned off. Signed-off-by: Matthew Arnold <[email protected]> * Avoid trying to delete non-existent storage class. If the test did not need to create an immediate-mode storage class (because the default was already set to immediate), make sure it does not try to delete anything after the test is done. Previously this would cause an error after all the tests were run, even if they were all successful. Signed-off-by: Matthew Arnold <[email protected]> * Create WFFC storage class if default is Immediate. Otherwise the previously-default WFFC tests will all run with Immediate, and make the Immediate-specific tests redundant. Signed-off-by: Matthew Arnold <[email protected]> * Add test scripts for OpenStack. Can't just use AWS for everything after all. Signed-off-by: Matthew Arnold <[email protected]> --------- Signed-off-by: Matthew Arnold <[email protected]>
openshift · Sep 13, 2024 · 510c203 · 510c203
1 parent fb62869
commit 510c203
Show file tree

Hide file tree

Showing 11 changed files with 214 additions and 25 deletions.
diff --git a/Makefile b/Makefile
@@ -30,6 +30,7 @@ VELERO_INSTANCE_NAME ?= velero-test
 ARTIFACT_DIR ?= /tmp
 OC_CLI = $(shell which oc)
 TEST_VIRT ?= false
+KVM_EMULATION ?= true
 TEST_UPGRADE ?= false
 
 ifdef CLI_DIR
@@ -63,6 +64,10 @@ ifeq ($(CLUSTER_TYPE), ibmcloud)
 	VELERO_PLUGIN = aws
 endif
 
+ifeq ($(CLUSTER_TYPE), openstack)
+	KVM_EMULATION = false
+endif
+
 # Kubernetes version from OpenShift 4.16.x https://openshift-release.apps.ci.l2s4.p1.openshiftapps.com/#4-stable
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
 ENVTEST_K8S_VERSION = 1.29
@@ -517,6 +522,7 @@ test-e2e: test-e2e-setup install-ginkgo
 	-velero_instance_name=$(VELERO_INSTANCE_NAME) \
 	-artifact_dir=$(ARTIFACT_DIR) \
 	-oc_cli=$(OC_CLI) \
+	-kvm_emulation=$(KVM_EMULATION) \
 	--ginkgo.vv \
 	--ginkgo.no-color=$(OPENSHIFT_CI) \
 	--ginkgo.label-filter="$(TEST_FILTER)" \

diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go
@@ -72,7 +72,7 @@ func prepareBackupAndRestore(brCase BackupRestoreCase, updateLastInstallTime fun
 	gomega.Eventually(dpaCR.BSLsAreAvailable(), time.Minute*3, time.Second*5).Should(gomega.BeTrue())
 
 	if brCase.BackupRestoreType == lib.CSI || brCase.BackupRestoreType == lib.CSIDataMover {
-		if provider == "aws" || provider == "ibmcloud" || provider == "gcp" || provider == "azure" {
+		if provider == "aws" || provider == "ibmcloud" || provider == "gcp" || provider == "azure" || provider == "openstack" {
 			log.Printf("Creating VolumeSnapshotClass for CSI backuprestore of %s", brCase.Name)
 			snapshotClassPath := fmt.Sprintf("./sample-applications/snapclass-csi/%s.yaml", provider)
 			err = lib.InstallApplication(dpaCR.Client, snapshotClassPath)

diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go
@@ -48,6 +48,8 @@ var (
 	kubeConfig          *rest.Config
 	knownFlake          bool
 	accumulatedTestLogs []string
+
+	kvmEmulation bool
 )
 
 func init() {
@@ -62,6 +64,7 @@ func init() {
 	flag.StringVar(&artifact_dir, "artifact_dir", "/tmp", "Directory for storing must gather")
 	flag.StringVar(&oc_cli, "oc_cli", "oc", "OC CLI Client")
 	flag.Int64Var(&flakeAttempts, "flakeAttempts", 3, "Customize the number of flake retries (3)")
+	flag.BoolVar(&kvmEmulation, "kvm_emulation", true, "Enable or disable KVM emulation for virtualization testing")
 
 	// helps with launching debug sessions from IDE
 	if os.Getenv("E2E_USE_ENV_FLAGS") == "true" {
@@ -100,7 +103,15 @@ func init() {
 				flakeAttempts = parsedValue
 			}
 		}
+		if envValue := os.Getenv("KVM_EMULATION"); envValue != "" {
+			if parsedValue, err := strconv.ParseBool(envValue); err == nil {
+				kvmEmulation = parsedValue
+			} else {
+				log.Println("Error parsing KVM_EMULATION, it will be enabled by default: ", err)
+			}
+		}
 	}
+
 }
 
 func TestOADPE2E(t *testing.T) {

diff --git a/tests/e2e/lib/virt_storage_helpers.go b/tests/e2e/lib/virt_storage_helpers.go
@@ -38,7 +38,7 @@ func (v *VirtOperator) deleteDataVolume(namespace, name string) error {
 	return v.Dynamic.Resource(dataVolumeGVR).Namespace(namespace).Delete(context.Background(), name, metav1.DeleteOptions{})
 }
 
-func (v *VirtOperator) checkDataVolumeExists(namespace, name string) bool {
+func (v *VirtOperator) CheckDataVolumeExists(namespace, name string) bool {
 	unstructuredDataVolume, err := v.getDataVolume(namespace, name)
 	if err != nil {
 		return false
@@ -122,7 +122,7 @@ func (v *VirtOperator) createDataVolumeFromUrl(namespace, name, url, size string
 
 // Create a DataVolume and wait for it to be ready.
 func (v *VirtOperator) EnsureDataVolumeFromUrl(namespace, name, url, size string, timeout time.Duration) error {
-	if !v.checkDataVolumeExists(namespace, name) {
+	if !v.CheckDataVolumeExists(namespace, name) {
 		if err := v.createDataVolumeFromUrl(namespace, name, url, size); err != nil {
 			return err
 		}
@@ -155,7 +155,7 @@ func (v *VirtOperator) RemoveDataVolume(namespace, name string, timeout time.Dur
 	}
 
 	err = wait.PollImmediate(5*time.Second, timeout, func() (bool, error) {
-		return !v.checkDataVolumeExists(namespace, name), nil
+		return !v.CheckDataVolumeExists(namespace, name), nil
 	})
 	if err != nil {
 		return fmt.Errorf("timed out waiting for DataVolume %s/%s to be deleted: %w", namespace, name, err)
@@ -207,28 +207,31 @@ func (v *VirtOperator) CreateDataSourceFromPvc(namespace, name string) error {
 	return nil
 }
 
-// Check the VolumeBindingMode of the default storage class, and make an
-// Immediate-mode copy if it is set to WaitForFirstConsumer.
-func (v *VirtOperator) CreateImmediateModeStorageClass(name string) error {
-	// Find the default storage class
+// Find the default storage class
+func (v *VirtOperator) GetDefaultStorageClass() (*storagev1.StorageClass, error) {
 	storageClasses, err := v.Clientset.StorageV1().StorageClasses().List(context.Background(), metav1.ListOptions{})
 	if err != nil {
-		return err
+		return nil, err
 	}
+
 	var defaultStorageClass *storagev1.StorageClass
 	for _, storageClass := range storageClasses.Items {
 		if storageClass.Annotations["storageclass.kubernetes.io/is-default-class"] == "true" {
 			log.Printf("Found default storage class: %s", storageClass.Name)
 			defaultStorageClass = storageClass.DeepCopy()
-			if storageClass.VolumeBindingMode != nil && *storageClass.VolumeBindingMode == storagev1.VolumeBindingImmediate {
-				log.Println("Default storage class already set to Immediate")
-				return nil
-			}
-			break
+			return defaultStorageClass, nil
 		}
 	}
-	if defaultStorageClass == nil {
-		return errors.New("no default storage class found")
+
+	return nil, errors.New("no default storage class found")
+}
+
+// Check the VolumeBindingMode of the default storage class, and make an
+// Immediate-mode copy if it is set to WaitForFirstConsumer.
+func (v *VirtOperator) CreateImmediateModeStorageClass(name string) error {
+	defaultStorageClass, err := v.GetDefaultStorageClass()
+	if err != nil {
+		return err
 	}
 
 	immediateStorageClass := defaultStorageClass
@@ -241,6 +244,31 @@ func (v *VirtOperator) CreateImmediateModeStorageClass(name string) error {
 	return err
 }
 
+// Check the VolumeBindingMode of the default storage class, and make a
+// WaitForFirstConsumer-mode copy if it is set to Immediate.
+func (v *VirtOperator) CreateWaitForFirstConsumerStorageClass(name string) error {
+	defaultStorageClass, err := v.GetDefaultStorageClass()
+	if err != nil {
+		return err
+	}
+
+	wffcStorageClass := defaultStorageClass
+	wffcStorageClass.VolumeBindingMode = ptr.To[storagev1.VolumeBindingMode](storagev1.VolumeBindingWaitForFirstConsumer)
+	wffcStorageClass.Name = name
+	wffcStorageClass.ResourceVersion = ""
+	wffcStorageClass.Annotations["storageclass.kubernetes.io/is-default-class"] = "false"
+
+	_, err = v.Clientset.StorageV1().StorageClasses().Create(context.Background(), wffcStorageClass, metav1.CreateOptions{})
+	return err
+}
+
 func (v *VirtOperator) RemoveStorageClass(name string) error {
-	return v.Clientset.StorageV1().StorageClasses().Delete(context.Background(), name, metav1.DeleteOptions{})
+	err := v.Clientset.StorageV1().StorageClasses().Delete(context.Background(), name, metav1.DeleteOptions{})
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return err
+	}
+	return nil
 }
diff --git a/tests/e2e/sample-applications/mongo-persistent/pvc/openstack.yaml b/tests/e2e/sample-applications/mongo-persistent/pvc/openstack.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: PersistentVolumeClaim
+    metadata:
+      name: mongo
+      namespace: mongo-persistent
+      labels:
+        app: mongo
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: ocs-storagecluster-ceph-rbd
+      resources:
+        requests:
+          storage: 1Gi
diff --git a/tests/e2e/sample-applications/mysql-persistent/pvc-twoVol/openstack.yaml b/tests/e2e/sample-applications/mysql-persistent/pvc-twoVol/openstack.yaml
@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: PersistentVolumeClaim
+    metadata:
+      name: mysql
+      namespace: mysql-persistent
+      labels:
+        app: mysql
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: ocs-storagecluster-ceph-rbd
+      resources:
+        requests:
+          storage: 1Gi
+  - apiVersion: v1
+    kind: PersistentVolumeClaim
+    metadata:
+      name: applog
+      namespace: mysql-persistent
+      labels:
+        app: todolist
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: ocs-storagecluster-ceph-rbd
+      resources:
+        requests:
+          storage: 1Gi
diff --git a/tests/e2e/sample-applications/mysql-persistent/pvc/openstack.yaml b/tests/e2e/sample-applications/mysql-persistent/pvc/openstack.yaml
@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: v1
+    kind: PersistentVolumeClaim
+    metadata:
+      name: mysql
+      namespace: mysql-persistent
+      labels:
+        app: mysql
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      storageClassName: ocs-storagecluster-ceph-rbd
+      resources:
+        requests:
+          storage: 1Gi
diff --git a/tests/e2e/sample-applications/snapclass-csi/openstack.yaml b/tests/e2e/sample-applications/snapclass-csi/openstack.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: List
+items:
+  - apiVersion: snapshot.storage.k8s.io/v1
+    kind: VolumeSnapshotClass
+    metadata:
+      name: oadp-example-snapclass
+      labels:
+        velero.io/csi-volumesnapshot-class: 'true'
+    driver: openshift-storage.rbd.csi.ceph.com
+    deletionPolicy: Retain
+    parameters:
+      clusterID: openshift-storage
+      csi.storage.k8s.io/snapshotter-secret-name: rook-csi-rbd-provisioner
+      csi.storage.k8s.io/snapshotter-secret-namespace: openshift-storage
diff --git a/tests/e2e/sample-applications/virtual-machines/cirros-test/cirros-test.yaml b/tests/e2e/sample-applications/virtual-machines/cirros-test/cirros-test.yaml
@@ -21,6 +21,7 @@ items:
               resources:
                 requests:
                   storage: 150Mi
+              storageClassName: test-sc-wffc
       running: true
       template:
         metadata:

diff --git a/tests/e2e/scripts/openstack_settings.sh b/tests/e2e/scripts/openstack_settings.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+cat > $TMP_DIR/oadpcreds <<EOF
+{
+  "spec": {
+      "configuration":{
+        "velero":{
+          "defaultPlugins": [
+            "openshift", "aws"
+          ]
+        }
+      },
+      "backupLocations": [
+        {
+          "velero": {
+            "provider": "aws",
+            "config": {
+             "profile": "$BSL_AWS_PROFILE",
+              "region": "$BSL_REGION"
+            },
+            "objectStorage":{
+              "bucket": "$BUCKET"
+            }
+          }
+        }
+      ],
+    "credential":{
+      "name": "$SECRET",
+      "key": "cloud"
+    },
+     "snapshotLocations": [
+       {
+         "velero": {
+           "provider": "aws",
+           "config": { 
+             "profile": "default",
+             "region": "$VSL_REGION"
+           }
+         }
+       }
+     ]
+  }
+}
+EOF
+
+x=$(cat $TMP_DIR/oadpcreds); echo "$x" | grep -o '^[^#]*'  > $TMP_DIR/oadpcreds
diff --git a/tests/e2e/virt_backup_restore_suite_test.go b/tests/e2e/virt_backup_restore_suite_test.go
@@ -153,6 +153,10 @@ var _ = ginkgo.Describe("VM backup and restore tests", ginkgo.Ordered, func() {
 	var v *lib.VirtOperator
 	var err error
 	wasInstalledFromTest := false
+
+	cirrosDownloadedFromTest := false
+	cirrosNamespace := "openshift-virtualization-os-images"
+
 	var lastBRCase VmBackupRestoreCase
 	var lastInstallTime time.Time
 	updateLastBRcase := func(brCase VmBackupRestoreCase) {
@@ -173,32 +177,45 @@ var _ = ginkgo.Describe("VM backup and restore tests", ginkgo.Ordered, func() {
 			wasInstalledFromTest = true
 		}
 
-		err = v.EnsureEmulation(20 * time.Second)
-		gomega.Expect(err).To(gomega.BeNil())
+		if kvmEmulation {
+			err = v.EnsureEmulation(20 * time.Second)
+			gomega.Expect(err).To(gomega.BeNil())
+		} else {
+			log.Println("Avoiding setting KVM emulation, by command line request")
+		}
 
 		url, err := getLatestCirrosImageURL()
 		gomega.Expect(err).To(gomega.BeNil())
-		err = v.EnsureDataVolumeFromUrl("openshift-virtualization-os-images", "cirros", url, "150Mi", 5*time.Minute)
-		gomega.Expect(err).To(gomega.BeNil())
-		err = v.CreateDataSourceFromPvc("openshift-virtualization-os-images", "cirros")
-		gomega.Expect(err).To(gomega.BeNil())
+		if !v.CheckDataVolumeExists(cirrosNamespace, "cirros") {
+			err = v.EnsureDataVolumeFromUrl(cirrosNamespace, "cirros", url, "150Mi", 5*time.Minute)
+			gomega.Expect(err).To(gomega.BeNil())
+			err = v.CreateDataSourceFromPvc(cirrosNamespace, "cirros")
+			gomega.Expect(err).To(gomega.BeNil())
+			cirrosDownloadedFromTest = true
+		}
 
 		dpaCR.VeleroDefaultPlugins = append(dpaCR.VeleroDefaultPlugins, v1alpha1.DefaultPluginKubeVirt)
 
 		err = v.CreateImmediateModeStorageClass("test-sc-immediate")
 		gomega.Expect(err).To(gomega.BeNil())
+		err = v.CreateWaitForFirstConsumerStorageClass("test-sc-wffc")
+		gomega.Expect(err).To(gomega.BeNil())
 	})
 
 	var _ = ginkgo.AfterAll(func() {
-		v.RemoveDataSource("openshift-virtualization-os-images", "cirros")
-		v.RemoveDataVolume("openshift-virtualization-os-images", "cirros", 2*time.Minute)
+		if v != nil && cirrosDownloadedFromTest {
+			v.RemoveDataSource(cirrosNamespace, "cirros")
+			v.RemoveDataVolume(cirrosNamespace, "cirros", 2*time.Minute)
+		}
 
 		if v != nil && wasInstalledFromTest {
 			v.EnsureVirtRemoval()
 		}
 
 		err := v.RemoveStorageClass("test-sc-immediate")
 		gomega.Expect(err).To(gomega.BeNil())
+		err = v.RemoveStorageClass("test-sc-wffc")
+		gomega.Expect(err).To(gomega.BeNil())
 	})
 
 	var _ = ginkgo.AfterEach(func(ctx ginkgo.SpecContext) {