diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c0468afd4..4da7084e0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,7 +17,65 @@ Developers must follow these steps to make a change: review process (effectively repeating steps 3-7 as needed). ## Developer Environment Installation -TODO + +### Cluster builds +In order to build on the cluster you need to first have your kubeconfig configured. Once configured you can run the following steps to build on the cluster: + +#### Configure the build + +```bash +$ make create-buildconfig +``` + +This will build from `https://github.com/openshift/lvm-operator` on branch `main` by default. This can be overridden by specifying the `GIT_URL` and `GIT_BRANCH` environment variables. +```bash +$ GIT_URL=https://github.com/my-user/lvm-operator.git \ +GIT_BRANCH=my-feature-branch \ +make create-buildconfig +``` + +#### Run the build +Kickoff the build on the cluster. All output will be followed for the build. +```bash +$ make cluster-build +``` + +#### Deploy the operator +To deploy the built operator run the following command: +```bash +$ make cluster-deploy +``` + +To undeploy the operator you can run +```bash +$ make undeploy +``` + + +### Local E2E Testing + +1. Download OpenShift Local from https://developers.redhat.com/products/openshift-local/overview +2. `crc setup` (once per machine) +3. `crc start` +4. ```shell + credentials=$(crc console --credentials -o json) + oc login -u $(echo $credentials | jq -r ".clusterConfig.adminCredentials.username") \ + -p $(echo $credentials | jq -r ".clusterConfig.adminCredentials.password") \ + $(echo $credentials | jq -r ".clusterConfig.url") + ``` +5. `oc config view --raw >> /tmp/crc-kubeconfig` +6. `export KUBECONFIG="/tmp/crc-kubeconfig"` +7. `make deploy` +8. `make e2e` + +#### Enable Snapshot Testing + +Prerequisites: Ensure you have a running CRC Cluster (Step 6) + +1. Make sure controller is undeployed with `make undeploy` +2. `oc apply -k https://github.com/kubernetes-csi/external-snapshotter//client/config/crd` +3. `oc apply -k https://github.com/kubernetes-csi/external-snapshotter//deploy/kubernetes/snapshot-controller` +4. Start again at Step 7 ## Commits Per Pull Request diff --git a/HACKING.md b/HACKING.md deleted file mode 100644 index aa362a4b0..000000000 --- a/HACKING.md +++ /dev/null @@ -1,34 +0,0 @@ -# Logical Volume Manager Storage Operator Hacking - -## Cluster builds -In order to build on the cluster you need to first have your kubeconfig configured. Once configured you can run the following steps to build on the cluster: - -### Configure the build - -```bash -$ make create-buildconfig -``` - -This will build from `https://github.com/openshift/lvm-operator` on branch `main` by default. This can be overridden by specifying the `GIT_URL` and `GIT_BRANCH` environment variables. -```bash -$ GIT_URL=https://github.com/my-user/lvm-operator.git \ -GIT_BRANCH=my-feature-branch \ -make create-buildconfig -``` - -### Run the build -Kickoff the build on the cluster. All output will be followed for the build. -```bash -$ make cluster-build -``` - -### Deploy the operator -To deploy the built operator run the following command: -```bash -$ make cluster-deploy -``` - -To undeploy the operator you can run -```bash -$ make undeploy -``` \ No newline at end of file diff --git a/Makefile b/Makefile index 7becf9856..009227c99 100644 --- a/Makefile +++ b/Makefile @@ -316,7 +316,7 @@ jsonnet: ## Download jsonnet locally if necessary. GINKGO = $(shell pwd)/bin/ginkgo ginkgo: ## Download ginkgo and gomega locally if necessary. - $(call go-get-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo@v2.9.4) + $(call go-get-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo@v2.9.5) # go-get-tool will 'go get' any package $2 and install it to $1. PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/controllers/lvmcluster_controller.go b/controllers/lvmcluster_controller.go index 1019ffca7..57457196b 100644 --- a/controllers/lvmcluster_controller.go +++ b/controllers/lvmcluster_controller.go @@ -177,7 +177,8 @@ func (r *LVMClusterReconciler) reconcile(ctx context.Context, instance *lvmv1alp } } if err != nil { - return ctrl.Result{Requeue: true, RequeueAfter: time.Minute * 1}, err + // check every 10 seconds if there are still PVCs present or the LogicalVolumes are removed + return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 10}, err } else { return reconcile.Result{}, nil } diff --git a/controllers/topolvm_snapshotclass.go b/controllers/topolvm_snapshotclass.go index 89fb1daed..dff54718f 100644 --- a/controllers/topolvm_snapshotclass.go +++ b/controllers/topolvm_snapshotclass.go @@ -18,13 +18,15 @@ package controllers import ( "context" + "errors" "fmt" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" lvmv1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" cutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -52,6 +54,11 @@ func (s topolvmVolumeSnapshotClass) ensureCreated(r *LVMClusterReconciler, ctx c // we anticipate no edits to volume snapshot class result, err := cutil.CreateOrUpdate(ctx, r.Client, vsc, func() error { return nil }) if err != nil { + // this is necessary in case the VolumeSnapshotClass CRDs are not registered in the Distro, e.g. for OpenShift Local + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + r.Log.Info("topolvm volume snapshot classes do not exist on the cluster, ignoring", "VolumeSnapshotClass", vscName) + return nil + } r.Log.Error(err, "topolvm volume snapshot class reconcile failure", "name", vsc.Name) return err } else { @@ -72,10 +79,15 @@ func (s topolvmVolumeSnapshotClass) ensureDeleted(r *LVMClusterReconciler, ctx c if err != nil { // already deleted in previous reconcile - if errors.IsNotFound(err) { + if k8serrors.IsNotFound(err) { r.Log.Info("topolvm volume snapshot class is deleted", "VolumeSnapshotClass", vscName) return nil } + // this is necessary in case the VolumeSnapshotClass CRDs are not registered in the Distro, e.g. for OpenShift Local + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + r.Log.Info("topolvm volume snapshot classes do not exist on the cluster, ignoring", "VolumeSnapshotClass", vscName) + return nil + } r.Log.Error(err, "failed to retrieve topolvm volume snapshot class", "VolumeSnapshotClass", vscName) return err } diff --git a/go.mod b/go.mod index 1b2fd4341..4b4aa97ed 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/aws/aws-sdk-go v1.44.10 github.com/go-logr/logr v1.2.4 + github.com/go-logr/zapr v1.2.4 github.com/google/go-cmp v0.5.9 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 github.com/onsi/ginkgo/v2 v2.9.5 @@ -18,6 +19,7 @@ require ( github.com/prometheus/client_golang v1.16.0 github.com/stretchr/testify v1.8.4 github.com/topolvm/topolvm v0.15.4-0.20221116041433-d58476400ff1 + go.uber.org/zap v1.24.0 gotest.tools/v3 v3.0.3 k8s.io/api v0.27.4 k8s.io/apimachinery v0.27.4 @@ -41,7 +43,6 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect @@ -82,7 +83,6 @@ require ( github.com/subosito/gotenv v1.2.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.24.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.10.0 // indirect diff --git a/test/e2e/config.go b/test/e2e/config.go index db117b421..1ef457b19 100644 --- a/test/e2e/config.go +++ b/test/e2e/config.go @@ -93,7 +93,7 @@ func getKubeconfig(kubeconfig string) (*rest.Config, error) { config, err = rest.InClusterConfig() } if err != nil { - return config, nil + return nil, err } return config, err } diff --git a/test/e2e/ephemeral_check.go b/test/e2e/ephemeral_check.go index 87a17f9ba..bb5d8d100 100644 --- a/test/e2e/ephemeral_check.go +++ b/test/e2e/ephemeral_check.go @@ -19,14 +19,17 @@ package e2e import ( "context" _ "embed" + "errors" "fmt" + "k8s.io/client-go/discovery" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" k8sv1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) var ( @@ -55,236 +58,256 @@ var ( func ephemeralTest() { Describe("Ephemeral Volume Tests", func() { var ( - pvc *k8sv1.PersistentVolumeClaim = &k8sv1.PersistentVolumeClaim{} - ephemeralPod *k8sv1.Pod - snapshot *snapapi.VolumeSnapshot - clonePvc *k8sv1.PersistentVolumeClaim - clonePod *k8sv1.Pod - restorePvc *k8sv1.PersistentVolumeClaim - restorePod *k8sv1.Pod - err error - ctx = context.Background() + pvc = &k8sv1.PersistentVolumeClaim{} + ephemeralPod *k8sv1.Pod + snapshot *snapapi.VolumeSnapshot + clonePvc *k8sv1.PersistentVolumeClaim + clonePod *k8sv1.Pod + restorePvc *k8sv1.PersistentVolumeClaim + restorePod *k8sv1.Pod + err error + skipSnapshotOps = false ) Context("Create ephemeral pod and volume", func() { - It("Tests ephemeral volume operations for VolumeMode=Filesystem", func() { - - By("Creating a pod with generic ephemeral volume") - podVolumeMountYaml := fmt.Sprintf(podEphemeralFSYAMLTemplate, "ephemeral-filepod", testNamespace, storageClassName) - ephemeralPod, err = getPod(podVolumeMountYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, ephemeralPod) - Expect(err).To(BeNil()) - - By("PVC should be bound") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-filepod-generic-ephemeral-volume", Namespace: testNamespace}, pvc) - return err == nil && pvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("PVC %s is bound\n", pvc.Name) - - By("Pod should be running") - Eventually(func() bool { - err = crClient.Get(ctx, types.NamespacedName{Name: ephemeralPod.Name, Namespace: testNamespace}, ephemeralPod) - return err == nil && ephemeralPod.Status.Phase == k8sv1.PodRunning - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Pod %s is running\n", ephemeralPod.Name) - - By("Creating a Snapshot of the pvc") - snapshotYaml := fmt.Sprintf(ephemeralVolumeSnapshotYAMLTemplate, "ephemeralfilepvc-snapshot", testNamespace, snapshotClass, "ephemeral-filepod-generic-ephemeral-volume") - snapshot, err = getVolumeSnapshot(snapshotYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is created\n", snapshot.Name) - - By("Verifying that the Snapshot is ready") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) - return err == nil && snapshot.Status != nil && *snapshot.Status.ReadyToUse - }, timeout, interval).Should(BeTrue()) - - By("Creating a clone of the pvc") - pvcCloneYaml := fmt.Sprintf(ephemeralPvcCloneYAMLTemplate, "ephemeralfilepvc-clone", testNamespace, "Filesystem", storageClassName, "ephemeral-filepod-generic-ephemeral-volume") - clonePvc, err = getPVC(pvcCloneYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Cloned PVC %s is created\n", clonePvc.Name) - - podVolumeMountYaml = fmt.Sprintf(podFSYAMLTemplate, "clone-ephemeralfilepod", testNamespace, "ephemeralfilepvc-clone") - clonePod, err = getPod(podVolumeMountYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: clonePvc.Name, Namespace: clonePvc.Namespace}, clonePvc) - return err == nil && clonePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Cloned PVC %s is bound\n", clonePvc.Name) - - By("Restore Snapshot for pvc") - pvcRestoreYaml := fmt.Sprintf(ephemeralPvcSnapshotRestoreYAMLTemplate, "ephemeralfilepvc-restore", testNamespace, "Filesystem", storageClassName, "ephemeralfilepvc-snapshot") - restorePvc, err = getPVC(pvcRestoreYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is restored\n", restorePvc.Name) - - podVolumeMountYaml = fmt.Sprintf(podFSYAMLTemplate, "restore-ephemeralfilepod", testNamespace, "ephemeralfilepvc-restore") - restorePod, err = getPod(podVolumeMountYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: restorePvc.Name, Namespace: restorePvc.Namespace}, restorePvc) - return err == nil && restorePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Restored PVC %s is bound\n", restorePvc.Name) - - err = crClient.Delete(ctx, clonePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", clonePod.Name) - - err = crClient.Delete(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Clone PVC %s is deleted\n", clonePvc.Name) - - err = crClient.Delete(ctx, restorePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", restorePod.Name) - - err = crClient.Delete(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Restored Snapshot %s is deleted\n", restorePvc.Name) - - err = crClient.Delete(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is deleted\n", snapshot.Name) - - By("Deleting the pod") - err = crClient.Delete(ctx, ephemeralPod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", ephemeralPod.Name) - - By("Confirming that ephemeral volume is automatically deleted") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-filepod-generic-ephemeral-volume", Namespace: testNamespace}, pvc) - return err != nil && errors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Deleting the pod, deleted the ephemeral volume %s\n", pvc.Name) - + Context("Tests ephemeral volume operations for VolumeMode=Filesystem", Ordered, func() { + It("Creation of ephemeral Pod binding a PVC from LVMS", func(ctx SpecContext) { + By("Creating an ephemeral pod") + podVolumeMountYaml := fmt.Sprintf(podEphemeralFSYAMLTemplate, "ephemeral-filepod", testNamespace, storageClassName) + ephemeralPod, err = getPod(podVolumeMountYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, ephemeralPod)).To(Succeed()) + + By("PVC should be bound") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-filepod-generic-ephemeral-volume", Namespace: testNamespace}, pvc); err != nil { + return err + } + if pvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", pvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Pod should be running") + Eventually(func(ctx context.Context) bool { + err = crClient.Get(ctx, types.NamespacedName{Name: ephemeralPod.Name, Namespace: testNamespace}, ephemeralPod) + return err == nil && ephemeralPod.Status.Phase == k8sv1.PodRunning + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + }) + + It("Testing Snapshot Operations", func(ctx SpecContext) { + By("Creating a Snapshot of the pvc") + snapshotYaml := fmt.Sprintf(ephemeralVolumeSnapshotYAMLTemplate, "ephemeralfilepvc-snapshot", testNamespace, snapshotClass, "ephemeral-filepod-generic-ephemeral-volume") + snapshot, err = getVolumeSnapshot(snapshotYaml) + Expect(err).To(BeNil()) + err = crClient.Create(ctx, snapshot) + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + skipSnapshotOps = true + Skip("Skipping Testing of Snapshot Operations due to lack of volume snapshot support") + } + Expect(err).To(BeNil()) + + By("Verifying that the Snapshot is ready") + Eventually(func() bool { + err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) + return err == nil && snapshot.Status != nil && *snapshot.Status.ReadyToUse + }, timeout, interval).Should(BeTrue()) + + By("Creating a clone of the pvc") + pvcCloneYaml := fmt.Sprintf(ephemeralPvcCloneYAMLTemplate, "ephemeralfilepvc-clone", testNamespace, "Filesystem", storageClassName, "ephemeral-filepod-generic-ephemeral-volume") + clonePvc, err = getPVC(pvcCloneYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, clonePvc)).To(Succeed()) + + By("Creating a pod consuming the clone of the pvc") + podVolumeMountYaml := fmt.Sprintf(podFSYAMLTemplate, "clone-ephemeralfilepod", testNamespace, "ephemeralfilepvc-clone") + clonePod, err = getPod(podVolumeMountYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, clonePod)).To(Succeed()) + + By("Having a bound claim in the pvc") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(clonePvc), clonePvc); err != nil { + return err + } + if clonePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", clonePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Restore Snapshot for pvc") + pvcRestoreYaml := fmt.Sprintf(ephemeralPvcSnapshotRestoreYAMLTemplate, "ephemeralfilepvc-restore", testNamespace, "Filesystem", storageClassName, "ephemeralfilepvc-snapshot") + restorePvc, err = getPVC(pvcRestoreYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, restorePvc)).To(Succeed()) + + By("Creating a pod consuming the restored snapshot data") + podVolumeMountYaml = fmt.Sprintf(podFSYAMLTemplate, "restore-ephemeralfilepod", testNamespace, "ephemeralfilepvc-restore") + restorePod, err = getPod(podVolumeMountYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, restorePod)).To(Succeed()) + + By("Having the restored data pvc be bound") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(restorePvc), restorePvc); err != nil { + return err + } + if restorePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", restorePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + }) + + It("Cleaning up ephemeral volume operations for VolumeMode=Filesystem", func(ctx SpecContext) { + if !skipSnapshotOps { + By(fmt.Sprintf("Deleting %s", clonePod.Name)) + Expect(crClient.Delete(ctx, clonePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Clone PVC %s", clonePvc.Name)) + Expect(crClient.Delete(ctx, clonePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting Pod %s", restorePod.Name)) + Expect(crClient.Delete(ctx, restorePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Snapshot PVC %s", restorePvc.Name)) + Expect(crClient.Delete(ctx, restorePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting VolumeSnapshot %s", snapshot.Name)) + Expect(crClient.Delete(ctx, snapshot)).To(Succeed()) + } + + By("Deleting Pod") + Expect(crClient.Delete(ctx, ephemeralPod)).To(Succeed()) + + By("Confirming that ephemeral volume is automatically deleted") + Eventually(func(ctx context.Context) bool { + err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-filepod-generic-ephemeral-volume", Namespace: testNamespace}, pvc) + return err != nil && k8serrors.IsNotFound(err) + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + }) }) - It("Tests PVC operations for VolumeMode=Block", func() { - By("Creating a pod with generic ephemeral volume") - podVolumeBlockYaml := fmt.Sprintf(podEphemeralBlockYAMLTemplate, "ephemeral-blockpod", testNamespace, storageClassName) - ephemeralPod, err = getPod(podVolumeBlockYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, ephemeralPod) - Expect(err).To(BeNil()) - - By("PVC should be bound") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-blockpod-generic-ephemeral-volume", Namespace: testNamespace}, pvc) - return err == nil && pvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("PVC %s is bound\n", pvc.Name) - - By("Pod should be running") - Eventually(func() bool { - err = crClient.Get(ctx, types.NamespacedName{Name: ephemeralPod.Name, Namespace: testNamespace}, ephemeralPod) - return err == nil && ephemeralPod.Status.Phase == k8sv1.PodRunning - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Pod %s is running\n", ephemeralPod.Name) - - By("Creating a Snapshot of the pvc") - snapshotYaml := fmt.Sprintf(ephemeralVolumeSnapshotYAMLTemplate, "ephemeralblockpvc-snapshot", testNamespace, snapshotClass, "ephemeral-blockpod-generic-ephemeral-volume") - snapshot, err = getVolumeSnapshot(snapshotYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is created\n", snapshot.Name) - - By("Verifying that the Snapshot is ready") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) - return err == nil && snapshot.Status != nil && *snapshot.Status.ReadyToUse - }, timeout, interval).Should(BeTrue()) - - By("Creating a clone of the pvc") - pvcCloneYaml := fmt.Sprintf(ephemeralPvcCloneYAMLTemplate, "ephemeralblockpvc-clone", testNamespace, "Block", storageClassName, "ephemeral-blockpod-generic-ephemeral-volume") - clonePvc, err = getPVC(pvcCloneYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Cloned PVC %s is created\n", clonePvc.Name) - - podVolumeBlockYaml = fmt.Sprintf(podBlockYAMLTemplate, "clone-ephemeralblockpod", testNamespace, "ephemeralblockpvc-clone") - clonePod, err = getPod(podVolumeBlockYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: clonePvc.Name, Namespace: clonePvc.Namespace}, clonePvc) - return err == nil && clonePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Cloned PVC %s is bound\n", clonePvc.Name) - - By("Restore Snapshot for pvc") - pvcRestoreYaml := fmt.Sprintf(ephemeralPvcSnapshotRestoreYAMLTemplate, "ephemeralblockpvc-restore", testNamespace, "Block", storageClassName, "ephemeralblockpvc-snapshot") - restorePvc, err = getPVC(pvcRestoreYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is restored\n", restorePvc.Name) - - podVolumeBlockYaml = fmt.Sprintf(podBlockYAMLTemplate, "restore-ephemeralblockpod", testNamespace, "ephemeralblockpvc-restore") - restorePod, err = getPod(podVolumeBlockYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: restorePvc.Name, Namespace: restorePvc.Namespace}, restorePvc) - return err == nil && restorePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Restored PVC %s is bound\n", restorePvc.Name) - - err = crClient.Delete(ctx, clonePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", clonePod.Name) - - err = crClient.Delete(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Clone PVC %s is deleted\n", clonePvc.Name) - - err = crClient.Delete(ctx, restorePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", restorePod.Name) - - err = crClient.Delete(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Restored Snapshot %s is deleted\n", restorePvc.Name) - - err = crClient.Delete(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is deleted\n", snapshot.Name) - - By("Deleting the pod") - err = crClient.Delete(ctx, ephemeralPod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", ephemeralPod.Name) - - By("Confirming that ephemeral volume is automatically deleted") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-blockpod-generic-ephemeral-volume", Namespace: testNamespace}, pvc) - return err != nil && errors.IsNotFound(err) - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Deleting the pod, deleted the ephemeral volume\n") + Context("Tests PVC operations for VolumeMode=Block", Ordered, func() { + It("Creation of ephemeral Pod binding a PVC from LVMS", func(ctx SpecContext) { + By("Creating an ephemeral pod") + podVolumeBlockYaml := fmt.Sprintf(podEphemeralBlockYAMLTemplate, "ephemeral-blockpod", testNamespace, storageClassName) + ephemeralPod, err = getPod(podVolumeBlockYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, ephemeralPod)).To(Succeed()) + + By("PVC should be bound") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-blockpod-generic-ephemeral-volume", Namespace: testNamespace}, pvc); err != nil { + return err + } + if pvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", pvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Pod should be running") + Eventually(func(ctx context.Context) bool { + err = crClient.Get(ctx, types.NamespacedName{Name: ephemeralPod.Name, Namespace: testNamespace}, ephemeralPod) + return err == nil && ephemeralPod.Status.Phase == k8sv1.PodRunning + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + }) + + It("Testing Snapshot Operations", func(ctx SpecContext) { + By("Creating a Snapshot of the pvc") + snapshotYaml := fmt.Sprintf(ephemeralVolumeSnapshotYAMLTemplate, "ephemeralblockpvc-snapshot", testNamespace, snapshotClass, "ephemeral-blockpod-generic-ephemeral-volume") + snapshot, err = getVolumeSnapshot(snapshotYaml) + Expect(err).To(BeNil()) + err = crClient.Create(ctx, snapshot) + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + skipSnapshotOps = true + Skip("Skipping Testing of Snapshot Operations due to lack of volume snapshot support") + } + Expect(err).To(BeNil()) + + By("Verifying that the Snapshot is ready") + Eventually(func(ctx context.Context) bool { + err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) + return err == nil && snapshot.Status != nil && *snapshot.Status.ReadyToUse + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + + By("Creating a clone of the pvc") + pvcCloneYaml := fmt.Sprintf(ephemeralPvcCloneYAMLTemplate, "ephemeralblockpvc-clone", testNamespace, "Block", storageClassName, "ephemeral-blockpod-generic-ephemeral-volume") + clonePvc, err = getPVC(pvcCloneYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, clonePvc)).To(BeNil()) + + By("Creating a pod consuming the clone of the pvc") + podVolumeBlockYaml := fmt.Sprintf(podBlockYAMLTemplate, "clone-ephemeralblockpod", testNamespace, "ephemeralblockpvc-clone") + clonePod, err = getPod(podVolumeBlockYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, clonePod)).To(BeNil()) + + By("Having a bound claim in the pvc") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(clonePvc), clonePvc); err != nil { + return err + } + if clonePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", clonePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Restore Snapshot for pvc") + pvcRestoreYaml := fmt.Sprintf(ephemeralPvcSnapshotRestoreYAMLTemplate, "ephemeralblockpvc-restore", testNamespace, "Block", storageClassName, "ephemeralblockpvc-snapshot") + restorePvc, err = getPVC(pvcRestoreYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, restorePvc)).To(Succeed()) + + By("Creating a pod consuming the restored snapshot data") + podVolumeBlockYaml = fmt.Sprintf(podBlockYAMLTemplate, "restore-ephemeralblockpod", testNamespace, "ephemeralblockpvc-restore") + restorePod, err = getPod(podVolumeBlockYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, restorePod)).To(Succeed()) + + By("Having the restored data pvc be bound") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(restorePvc), restorePvc); err != nil { + return err + } + if restorePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", restorePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + }) + + It("Cleaning up ephemeral volume operations for VolumeMode=Filesystem", func(ctx SpecContext) { + if !skipSnapshotOps { + By(fmt.Sprintf("Deleting %s", clonePod.Name)) + Expect(crClient.Delete(ctx, clonePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Clone PVC %s", clonePvc.Name)) + Expect(crClient.Delete(ctx, clonePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting Pod %s", restorePod.Name)) + Expect(crClient.Delete(ctx, restorePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Snapshot PVC %s", restorePvc.Name)) + Expect(crClient.Delete(ctx, restorePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting VolumeSnapshot %s", snapshot.Name)) + Expect(crClient.Delete(ctx, snapshot)).To(Succeed()) + } + + By("Deleting Pod") + Expect(crClient.Delete(ctx, ephemeralPod)).To(Succeed()) + + By("Confirming that ephemeral volume is automatically deleted") + Eventually(func(ctx context.Context) bool { + err := crClient.Get(ctx, types.NamespacedName{Name: "ephemeral-blockpod-generic-ephemeral-volume", Namespace: testNamespace}, pvc) + return err != nil && k8serrors.IsNotFound(err) + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + }) }) }) diff --git a/test/e2e/helper.go b/test/e2e/helper.go index 1d968a45e..52cf0150a 100644 --- a/test/e2e/helper.go +++ b/test/e2e/helper.go @@ -21,7 +21,6 @@ import ( "fmt" "time" - "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" k8sv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -30,11 +29,6 @@ import ( utilwait "k8s.io/apimachinery/pkg/util/wait" ) -// debug immediately prints message in ginkgo test suite -func debug(args ...interface{}) { - ginkgo.GinkgoWriter.Println(args) -} - // createNamespace creates a namespace in the cluster, ignoring if it already exists. func createNamespace(ctx context.Context, namespace string) error { label := make(map[string]string) diff --git a/test/e2e/lvm_suite_test.go b/test/e2e/lvm_suite_test.go index d7b2d1231..cc3464086 100644 --- a/test/e2e/lvm_suite_test.go +++ b/test/e2e/lvm_suite_test.go @@ -19,7 +19,13 @@ package e2e import ( "context" "flag" + "github.com/go-logr/zapr" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "sigs.k8s.io/controller-runtime/pkg/log" + ctrlZap "sigs.k8s.io/controller-runtime/pkg/log/zap" "testing" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,6 +39,15 @@ func TestLvm(t *testing.T) { } var _ = BeforeSuite(func() { + core := zapcore.NewCore( + &ctrlZap.KubeAwareEncoder{Encoder: zapcore.NewConsoleEncoder(zap.NewDevelopmentEncoderConfig())}, + zapcore.AddSync(GinkgoWriter), + zap.NewAtomicLevelAt(zapcore.Level(-9)), + ) + zapLog := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel)) + logr := zapr.NewLogger(zapLog.With(zap.Namespace("context"))) + log.SetLogger(logr) + // Configure the disk and install the operator beforeTestSuiteSetup(context.Background()) lvmNamespaceSetup(context.Background()) @@ -50,9 +65,11 @@ var _ = Describe("LVM Operator e2e tests", func() { // Ordered to give the BeforeAll/AfterAll functionality to achieve common setup var clusterConfig *v1alpha1.LVMCluster - BeforeAll(func() { + BeforeAll(func(ctx SpecContext) { clusterConfig = generateLVMCluster() - lvmClusterSetup(clusterConfig, context.Background()) + lvmClusterSetup(clusterConfig, ctx) + By("Verifying the cluster is ready") + Eventually(clusterReadyCheck(clusterConfig), timeout, 300*time.Millisecond).WithContext(ctx).Should(Succeed()) }) Describe("Functional Tests", func() { @@ -61,8 +78,8 @@ var _ = Describe("LVM Operator e2e tests", func() { Context("Ephemeral volume tests", ephemeralTest) }) - AfterAll(func() { - lvmClusterCleanup(clusterConfig, context.Background()) + AfterAll(func(ctx SpecContext) { + lvmClusterCleanup(clusterConfig, ctx) }) }) }) diff --git a/test/e2e/lvmcluster.go b/test/e2e/lvmcluster.go index 3b009d61f..98f91a579 100644 --- a/test/e2e/lvmcluster.go +++ b/test/e2e/lvmcluster.go @@ -18,19 +18,18 @@ package e2e import ( "context" - "fmt" + "errors" "time" - v1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" - corev1 "k8s.io/api/core/v1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/openshift/lvm-operator/api/v1alpha1" storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" utilwait "k8s.io/apimachinery/pkg/util/wait" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" ) const ( @@ -69,9 +68,7 @@ func startLVMCluster(clusterConfig *v1alpha1.LVMCluster, ctx context.Context) er // deleteLVMCluster deletes a sample CR. func deleteLVMCluster(clusterConfig *v1alpha1.LVMCluster, ctx context.Context) error { - lvmClusterRes := generateLVMCluster() - cluster := &v1alpha1.LVMCluster{} - err := crClient.Delete(ctx, lvmClusterRes) + err := crClient.Delete(ctx, clusterConfig) if err != nil { return err } @@ -80,9 +77,9 @@ func deleteLVMCluster(clusterConfig *v1alpha1.LVMCluster, ctx context.Context) e interval := 10 * time.Second // wait for LVMCluster to be deleted - err = utilwait.PollImmediate(interval, timeout, func() (done bool, err error) { - err = crClient.Get(ctx, types.NamespacedName{Name: lvmClusterRes.Name, Namespace: installNamespace}, cluster) - if err != nil && errors.IsNotFound(err) { + err = utilwait.PollUntilContextTimeout(ctx, interval, timeout, true, func(ctx context.Context) (done bool, err error) { + err = crClient.Get(ctx, types.NamespacedName{Name: clusterConfig.Name, Namespace: installNamespace}, clusterConfig) + if err != nil && k8serrors.IsNotFound(err) { return true, nil } if err == nil { @@ -94,114 +91,95 @@ func deleteLVMCluster(clusterConfig *v1alpha1.LVMCluster, ctx context.Context) e return err } -func setupPodAndPVC() (*corev1.PersistentVolumeClaim, *corev1.Pod) { - filePvcYaml := fmt.Sprintf(pvcYAMLTemplate, "lvmfilepvc", testNamespace, "Filesystem", storageClassName) - pvc, err := getPVC(filePvcYaml) - Expect(err).To(BeNil()) - - err = crClient.Create(context.Background(), pvc) - Expect(err).To(BeNil()) - - podVolumeMountYaml := fmt.Sprintf(podVolumeFSYAMLTemplate, "lvmfilepod", testNamespace, "lvmfilepvc") - pod, err := getPod(podVolumeMountYaml) - Expect(err).To(BeNil()) - - err = crClient.Create(context.Background(), pod) - Expect(err).To(BeNil()) - - return pvc, pod -} - -func cleanupPVCAndPod(pvc *corev1.PersistentVolumeClaim, pod *corev1.Pod) { - err := crClient.Delete(context.Background(), pod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", pod.Name) - - err = crClient.Delete(context.Background(), pvc) - Expect(err).To(BeNil()) - fmt.Printf("PVC %s is deleted\n", pvc.Name) -} - func lvmClusterTest() { Describe("Filesystem Type", Serial, func() { var clusterConfig *v1alpha1.LVMCluster - ctx := context.Background() - AfterEach(func() { + AfterEach(func(ctx SpecContext) { // Delete the cluster lvmClusterCleanup(clusterConfig, ctx) }) - It("should default to xfs", func() { + It("should default to xfs", func(ctx SpecContext) { clusterConfig = generateLVMCluster() // Do not specify a fstype By("Setting up the cluster with the default fstype") lvmClusterSetup(clusterConfig, ctx) + By("Verifying the cluster is ready") + Eventually(clusterReadyCheck(clusterConfig), timeout, 300*time.Millisecond).WithContext(ctx).Should(Succeed()) + + By("Checking that the Storage Class is created normally") // Make sure the storage class was configured properly sc := storagev1.StorageClass{} - var err error - Eventually(func() bool { - err = crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &sc) - - return err == nil - }, timeout, interval).Should(BeTrue()) + By("Verifying the StorageClass exists") + Eventually(func(ctx SpecContext) error { + return crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &sc) + }, timeout, interval).WithContext(ctx).Should(Succeed()) - if err != nil { - Fail(fmt.Sprintf("Error getting StorageClass %s: %s\n", storageClassName, err.Error())) - } + By("Verifying that the default FS type is set to XFS on the StorageClass") Expect(sc.Parameters["csi.storage.k8s.io/fstype"]).To(Equal(string(v1alpha1.FilesystemTypeXFS))) }) - It("should be xfs if specified", func() { + It("should be xfs if specified", func(ctx SpecContext) { clusterConfig = generateLVMCluster() clusterConfig.Spec.Storage.DeviceClasses[0].FilesystemType = v1alpha1.FilesystemTypeXFS - By("Setting up the cluster with the default fstype") + By("Setting up the cluster with xfs fstype") lvmClusterSetup(clusterConfig, ctx) + By("Verifying the cluster is ready") + Eventually(clusterReadyCheck(clusterConfig), timeout, 300*time.Millisecond).WithContext(ctx).Should(Succeed()) + // Make sure the storage class was configured properly sc := storagev1.StorageClass{} - var err error - Eventually(func() bool { - err = crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &sc) - - return err == nil - }, timeout, interval).Should(BeTrue()) - - if err != nil { - Fail(fmt.Sprintf("Error getting StorageClass %s: %s\n", storageClassName, err.Error())) - } + By("Verifying the StorageClass exists") + Eventually(func(ctx SpecContext) error { + return crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &sc) + }, timeout, interval).WithContext(ctx).Should(Succeed()) + By("Verifying the correct fstype Parameter") Expect(sc.Parameters["csi.storage.k8s.io/fstype"]).To(Equal(string(v1alpha1.FilesystemTypeXFS))) }) - It("should be ext4 if specified", func() { + It("should be ext4 if specified", func(ctx SpecContext) { clusterConfig = generateLVMCluster() clusterConfig.Spec.Storage.DeviceClasses[0].FilesystemType = v1alpha1.FilesystemTypeExt4 By("Setting up the cluster with the ext4 fstype") lvmClusterSetup(clusterConfig, ctx) + By("Verifying the cluster is ready") + Eventually(clusterReadyCheck(clusterConfig), timeout, 300*time.Millisecond).WithContext(ctx).Should(Succeed()) + // Make sure the storage class was configured properly sc := storagev1.StorageClass{} - var err error - Eventually(func() bool { - err = crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &sc) - - return err == nil - }, timeout, interval).Should(BeTrue()) - - if err != nil { - Fail(fmt.Sprintf("Error getting StorageClass %s: %s\n", storageClassName, err.Error())) - } + By("Verifying the StorageClass exists") + Eventually(func(ctx SpecContext) error { + return crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &sc) + }, timeout, interval).WithContext(ctx).Should(Succeed()) + By("Verifying the correct fstype Parameter") Expect(sc.Parameters["csi.storage.k8s.io/fstype"]).To(Equal(string(v1alpha1.FilesystemTypeExt4))) }) }) } + +func clusterReadyCheck(clusterConfig *v1alpha1.LVMCluster) func(ctx context.Context) error { + return func(ctx context.Context) error { + currentCluster := clusterConfig + err := crClient.Get(ctx, client.ObjectKeyFromObject(clusterConfig), currentCluster) + if err != nil { + return err + } + if currentCluster.Status.State == v1alpha1.LVMStatusReady { + return nil + } + return errors.New("cluster is not ready yet") + } +} diff --git a/test/e2e/pvc_check.go b/test/e2e/pvc_check.go index e23e1e86e..2c0102db2 100644 --- a/test/e2e/pvc_check.go +++ b/test/e2e/pvc_check.go @@ -19,13 +19,16 @@ package e2e import ( "context" _ "embed" + "errors" "fmt" + "k8s.io/client-go/discovery" snapapi "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" k8sv1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) //go:embed testdata/pvc_tests/pvc-template.yaml @@ -57,234 +60,264 @@ func pvcTest() { var restorePvc *k8sv1.PersistentVolumeClaim var restorePod *k8sv1.Pod var err error - ctx := context.Background() - - Context("create pvc, pod, snapshots, clones", func() { - It("Tests PVC operations for VolumeMode=Filesystem", func() { - By("Creating a pvc and pod") - filePvcYaml := fmt.Sprintf(pvcYAMLTemplate, "lvmfilepvc", testNamespace, "Filesystem", storageClassName) - pvc, err = getPVC(filePvcYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, pvc) - Expect(err).To(BeNil()) - - podVolumeMountYaml := fmt.Sprintf(podVolumeFSYAMLTemplate, "lvmfilepod", testNamespace, "lvmfilepvc") - pod, err = getPod(podVolumeMountYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, pod) - Expect(err).To(BeNil()) - - By("Verifying that the PVC(file system) is bound and the Pod is running") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, pvc) - return err == nil && pvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("PVC %s is bound\n", pvc.Name) - - Eventually(func() bool { - err = crClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, pod) - return err == nil && pod.Status.Phase == k8sv1.PodRunning - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Pod %s is running\n", pod.Name) - - By("Creating a Snapshot of the file-pvc") - snapshotYaml := fmt.Sprintf(volumeSnapshotYAMLTemplate, "lvmfilepvc-snapshot", testNamespace, snapshotClass, "lvmfilepvc") - snapshot, err = getVolumeSnapshot(snapshotYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is created\n", snapshot.Name) - - By("Verifying that the Snapshot is ready") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) - if err == nil && snapshot.Status != nil && snapshot.Status.ReadyToUse != nil { - return *snapshot.Status.ReadyToUse + + var skipSnapshotOps bool + + Context("create pvc, pod, snapshots, clones", Ordered, func() { + Context("Tests PVC operations for VolumeMode=Filesystem", Ordered, func() { + It("Creation of Pod binding a PVC from LVMS", func(ctx SpecContext) { + By("Creating a pvc") + filePvcYaml := fmt.Sprintf(pvcYAMLTemplate, "lvmfilepvc", testNamespace, "Filesystem", storageClassName) + pvc, err = getPVC(filePvcYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, pvc)).To(Succeed()) + + By("Creating a pod") + podVolumeMountYaml := fmt.Sprintf(podVolumeFSYAMLTemplate, "lvmfilepod", testNamespace, "lvmfilepvc") + pod, err = getPod(podVolumeMountYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, pod)).To(Succeed()) + + By("Verifying that the PVC(file system) is bound and the Pod is running") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, pvc); err != nil { + return err + } + if pvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", pvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Pod should be running") + Eventually(func(ctx context.Context) bool { + err = crClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, pod) + return err == nil && pod.Status.Phase == k8sv1.PodRunning + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + }) + + It("Testing Snapshot Operations", func(ctx SpecContext) { + By("Creating a Snapshot of the file-pvc") + snapshotYaml := fmt.Sprintf(volumeSnapshotYAMLTemplate, "lvmfilepvc-snapshot", testNamespace, snapshotClass, "lvmfilepvc") + snapshot, err = getVolumeSnapshot(snapshotYaml) + Expect(err).To(BeNil()) + err = crClient.Create(ctx, snapshot) + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + skipSnapshotOps = true + Skip("Skipping Testing of Snapshot Operations due to lack of volume snapshot support") + } + Expect(err).To(BeNil()) + + By("Verifying that the Snapshot is ready") + Eventually(func(ctx context.Context) bool { + err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) + if err == nil && snapshot.Status != nil && snapshot.Status.ReadyToUse != nil { + return *snapshot.Status.ReadyToUse + } + return false + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + + By("Creating a clone of the filesystem pvc") + pvcCloneYaml := fmt.Sprintf(pvcCloneYAMLTemplate, "lvmfilepvc-clone", testNamespace, "Filesystem", storageClassName, "lvmfilepvc") + clonePvc, err = getPVC(pvcCloneYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, clonePvc)).To(Succeed()) + + By("Creating a pod consuming the clone") + podVolumeMountYaml := fmt.Sprintf(podVolumeFSYAMLTemplate, "clone-lvmfilepod", testNamespace, "lvmfilepvc-clone") + clonePod, err = getPod(podVolumeMountYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, clonePod)).To(Succeed()) + + By("Having a bound claim in the pvc") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(clonePvc), clonePvc); err != nil { + return err + } + if clonePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", clonePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Restore Snapshot for file-pvc") + pvcRestoreYaml := fmt.Sprintf(pvcSnapshotRestoreYAMLTemplate, "lvmfilepvc-restore", testNamespace, "Filesystem", storageClassName, "lvmfilepvc-snapshot") + restorePvc, err = getPVC(pvcRestoreYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, restorePvc)).To(Succeed()) + + By("Creating a pod consuming the restored snapshot of the pvc") + podVolumeMountYaml = fmt.Sprintf(podVolumeFSYAMLTemplate, "restore-lvmfilepod", testNamespace, "lvmfilepvc-restore") + restorePod, err = getPod(podVolumeMountYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, restorePod)).To(Succeed()) + + By("Having the restored data pvc be bound") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(restorePvc), restorePvc); err != nil { + return err + } + if restorePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", restorePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + }) + + It("Cleaning up for VolumeMode=Filesystem", func(ctx SpecContext) { + if !skipSnapshotOps { + By(fmt.Sprintf("Deleting %s", clonePod.Name)) + Expect(crClient.Delete(ctx, clonePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Clone PVC %s", clonePvc.Name)) + Expect(crClient.Delete(ctx, clonePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting Pod %s", restorePod.Name)) + Expect(crClient.Delete(ctx, restorePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Snapshot PVC %s", restorePvc.Name)) + Expect(crClient.Delete(ctx, restorePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting VolumeSnapshot %s", snapshot.Name)) + Expect(crClient.Delete(ctx, snapshot)).To(Succeed()) } - return false - }, timeout, interval).Should(BeTrue()) - - By("Creating a clone of the filesystem pvc") - pvcCloneYaml := fmt.Sprintf(pvcCloneYAMLTemplate, "lvmfilepvc-clone", testNamespace, "Filesystem", storageClassName, "lvmfilepvc") - clonePvc, err = getPVC(pvcCloneYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Cloned PVC %s is created\n", clonePvc.Name) - - podVolumeMountYaml = fmt.Sprintf(podVolumeFSYAMLTemplate, "clone-lvmfilepod", testNamespace, "lvmfilepvc-clone") - clonePod, err = getPod(podVolumeMountYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: clonePvc.Name, Namespace: clonePvc.Namespace}, clonePvc) - return err == nil && clonePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Cloned PVC %s is bound\n", clonePvc.Name) - - By("Restore Snapshot for file-pvc") - pvcRestoreYaml := fmt.Sprintf(pvcSnapshotRestoreYAMLTemplate, "lvmfilepvc-restore", testNamespace, "Filesystem", storageClassName, "lvmfilepvc-snapshot") - restorePvc, err = getPVC(pvcRestoreYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is restored\n", restorePvc.Name) - - podVolumeMountYaml = fmt.Sprintf(podVolumeFSYAMLTemplate, "restore-lvmfilepod", testNamespace, "lvmfilepvc-restore") - restorePod, err = getPod(podVolumeMountYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: restorePvc.Name, Namespace: restorePvc.Namespace}, restorePvc) - return err == nil && restorePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Restored PVC %s is bound\n", restorePvc.Name) - - err = crClient.Delete(ctx, clonePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", clonePod.Name) - - err = crClient.Delete(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Clone PVC %s is deleted\n", clonePvc.Name) - - err = crClient.Delete(ctx, restorePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", restorePod.Name) - - err = crClient.Delete(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Restored Snapshot %s is deleted\n", restorePvc.Name) - - err = crClient.Delete(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is deleted\n", snapshot.Name) - - err = crClient.Delete(ctx, pod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", pod.Name) - - err = crClient.Delete(ctx, pvc) - Expect(err).To(BeNil()) - fmt.Printf("PVC %s is deleted\n", pvc.Name) + + By(fmt.Sprintf("Deleting Pod %s", pod.Name)) + Expect(crClient.Delete(ctx, pod)).To(Succeed()) + + By(fmt.Sprintf("Deleting PVC %s", pvc.Name)) + Expect(crClient.Delete(ctx, pvc)).To(Succeed()) + }) }) - It("Tests PVC operations for VolumeMode=Block", func() { - By("Creating pvc and pod") - blockPvcYaml := fmt.Sprintf(pvcYAMLTemplate, "lvmblockpvc", testNamespace, "Block", storageClassName) - pvc, err = getPVC(blockPvcYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, pvc) - Expect(err).To(BeNil()) - - podVolumeBlockYaml := fmt.Sprintf(podVolumeBlockYAMLTemplate, "lvmblockpod", testNamespace, "lvmblockpvc") - pod, err = getPod(podVolumeBlockYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, pod) - Expect(err).To(BeNil()) - - By("Verifying that the PVC(block) is bound and the Pod is running") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: pvc.Name, Namespace: pvc.Namespace}, pvc) - return err == nil && pvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("PVC %s is bound\n", pvc.Name) - - Eventually(func() bool { - err = crClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, pod) - return err == nil && pod.Status.Phase == k8sv1.PodRunning - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Pod %s is running\n", pod.Name) - - By("Creating a Snapshot of the block-pvc") - snapshotYaml := fmt.Sprintf(volumeSnapshotYAMLTemplate, "lvmblockpvc-snapshot", testNamespace, snapshotClass, "lvmblockpvc") - snapshot, err = getVolumeSnapshot(snapshotYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is created\n", snapshot.Name) - - By("Verifying that the Snapshot is ready") - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) - if err == nil && snapshot.Status != nil && snapshot.Status.ReadyToUse != nil { - return *snapshot.Status.ReadyToUse + Context("Tests PVC operations for VolumeMode=Block", Ordered, func() { + It("Creation of Pod binding a PVC from LVMS", func(ctx SpecContext) { + By("Creating a pvc") + blockPvcYaml := fmt.Sprintf(pvcYAMLTemplate, "lvmblockpvc", testNamespace, "Block", storageClassName) + pvc, err = getPVC(blockPvcYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, pvc)).To(Succeed()) + + By("Creating a pod") + podVolumeBlockYaml := fmt.Sprintf(podVolumeBlockYAMLTemplate, "lvmblockpod", testNamespace, "lvmblockpvc") + pod, err = getPod(podVolumeBlockYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, pod)).To(Succeed()) + + By("Verifying that the PVC(block) is bound and the Pod is running") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(pvc), pvc); err != nil { + return err + } + if pvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", pvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Pod should be running") + Eventually(func(ctx context.Context) bool { + err = crClient.Get(ctx, types.NamespacedName{Name: pod.Name, Namespace: pod.Namespace}, pod) + return err == nil && pod.Status.Phase == k8sv1.PodRunning + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + }) + + It("Testing Snapshot Operations", func(ctx SpecContext) { + By("Creating a Snapshot of the block-pvc") + snapshotYaml := fmt.Sprintf(volumeSnapshotYAMLTemplate, "lvmblockpvc-snapshot", testNamespace, snapshotClass, "lvmblockpvc") + snapshot, err = getVolumeSnapshot(snapshotYaml) + Expect(err).To(BeNil()) + err = crClient.Create(ctx, snapshot) + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + skipSnapshotOps = true + Skip("Skipping Testing of Snapshot Operations due to lack of volume snapshot support") } - return false - }, timeout, interval).Should(BeTrue()) - - By("Creating a clone of the block-pvc") - pvcCloneYaml := fmt.Sprintf(pvcCloneYAMLTemplate, "lvmblockpvc-clone", testNamespace, "Block", storageClassName, "lvmblockpvc") - clonePvc, err = getPVC(pvcCloneYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Cloned PVC %s is created\n", clonePvc.Name) - - podVolumeBlockYaml = fmt.Sprintf(podVolumeBlockYAMLTemplate, "clone-lvmblockpod", testNamespace, "lvmblockpvc-clone") - clonePod, err = getPod(podVolumeBlockYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, clonePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: clonePvc.Name, Namespace: clonePvc.Namespace}, clonePvc) - return err == nil && clonePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Cloned PVC %s is bound\n", clonePvc.Name) - - By("Restore Snapshot for block-pvc") - pvcRestoreYaml := fmt.Sprintf(pvcSnapshotRestoreYAMLTemplate, "lvmblockpvc-restore", testNamespace, "Block", storageClassName, "lvmblockpvc-snapshot") - restorePvc, err = getPVC(pvcRestoreYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is restored\n", restorePvc.Name) - - podVolumeBlockYaml = fmt.Sprintf(podVolumeBlockYAMLTemplate, "restore-lvmblockpod", testNamespace, "lvmblockpvc-restore") - restorePod, err = getPod(podVolumeBlockYaml) - Expect(err).To(BeNil()) - err = crClient.Create(ctx, restorePod) - Expect(err).To(BeNil()) - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: restorePvc.Name, Namespace: restorePvc.Namespace}, restorePvc) - return err == nil && restorePvc.Status.Phase == k8sv1.ClaimBound - }, timeout, interval).Should(BeTrue()) - fmt.Printf("Restored PVC %s is bound\n", restorePod.Name) - - err = crClient.Delete(ctx, clonePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", clonePod.Name) - - err = crClient.Delete(ctx, clonePvc) - Expect(err).To(BeNil()) - fmt.Printf("Clone PVC %s is deleted\n", clonePvc.Name) - - err = crClient.Delete(ctx, restorePod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", restorePod.Name) - - err = crClient.Delete(ctx, restorePvc) - Expect(err).To(BeNil()) - fmt.Printf("Restored Snapshot %s is deleted\n", restorePvc.Name) - - err = crClient.Delete(ctx, snapshot) - Expect(err).To(BeNil()) - fmt.Printf("Snapshot %s is deleted\n", snapshot.Name) - - err = crClient.Delete(ctx, pod) - Expect(err).To(BeNil()) - fmt.Printf("Pod %s is deleted\n", pod.Name) - - err = crClient.Delete(ctx, pvc) - Expect(err).To(BeNil()) - fmt.Printf("PVC %s is deleted\n", pvc.Name) + Expect(err).To(BeNil()) + + By("Verifying that the Snapshot is ready") + Eventually(func(ctx context.Context) bool { + err := crClient.Get(ctx, types.NamespacedName{Name: snapshot.Name, Namespace: snapshot.Namespace}, snapshot) + if err == nil && snapshot.Status != nil && snapshot.Status.ReadyToUse != nil { + return *snapshot.Status.ReadyToUse + } + return false + }, timeout, interval).WithContext(ctx).Should(BeTrue()) + + By("Creating a clone of the block-pvc") + pvcCloneYaml := fmt.Sprintf(pvcCloneYAMLTemplate, "lvmblockpvc-clone", testNamespace, "Block", storageClassName, "lvmblockpvc") + clonePvc, err = getPVC(pvcCloneYaml) + Expect(err).To(BeNil()) + err = crClient.Create(ctx, clonePvc) + Expect(err).To(BeNil()) + + By("Creating a pod consuming the clone of the pvc") + podVolumeBlockYaml := fmt.Sprintf(podVolumeBlockYAMLTemplate, "clone-lvmblockpod", testNamespace, "lvmblockpvc-clone") + clonePod, err = getPod(podVolumeBlockYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, clonePod)).To(Succeed()) + + By("Having a bound claim in the pvc") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(clonePvc), clonePvc); err != nil { + return err + } + if clonePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", clonePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + + By("Restore Snapshot for block-pvc") + pvcRestoreYaml := fmt.Sprintf(pvcSnapshotRestoreYAMLTemplate, "lvmblockpvc-restore", testNamespace, "Block", storageClassName, "lvmblockpvc-snapshot") + restorePvc, err = getPVC(pvcRestoreYaml) + Expect(err).To(BeNil()) + err = crClient.Create(ctx, restorePvc) + Expect(err).To(BeNil()) + + By("Creating a pod consuming the restored snapshot data") + podVolumeBlockYaml = fmt.Sprintf(podVolumeBlockYAMLTemplate, "restore-lvmblockpod", testNamespace, "lvmblockpvc-restore") + restorePod, err = getPod(podVolumeBlockYaml) + Expect(err).To(BeNil()) + Expect(crClient.Create(ctx, restorePod)).To(Succeed()) + + By("Having the restored data pvc be bound") + Eventually(func(ctx context.Context) error { + if err := crClient.Get(ctx, client.ObjectKeyFromObject(restorePvc), restorePvc); err != nil { + return err + } + if restorePvc.Status.Phase != k8sv1.ClaimBound { + return fmt.Errorf("pvc is not bound yet: %s", restorePvc.Status.Phase) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) + }) + + It("Cleaning up for VolumeMode=Block", func(ctx SpecContext) { + if !skipSnapshotOps { + By(fmt.Sprintf("Deleting %s", clonePod.Name)) + Expect(crClient.Delete(ctx, clonePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Clone PVC %s", clonePvc.Name)) + Expect(crClient.Delete(ctx, clonePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting Pod %s", restorePod.Name)) + Expect(crClient.Delete(ctx, restorePod)).To(Succeed()) + + By(fmt.Sprintf("Deleting Snapshot PVC %s", restorePvc.Name)) + Expect(crClient.Delete(ctx, restorePvc)).To(Succeed()) + + By(fmt.Sprintf("Deleting VolumeSnapshot %s", snapshot.Name)) + Expect(crClient.Delete(ctx, snapshot)).To(Succeed()) + } + + By(fmt.Sprintf("Deleting Pod %s", pod.Name)) + Expect(crClient.Delete(ctx, pod)).To(Succeed()) + + By(fmt.Sprintf("Deleting PVC %s", pvc.Name)) + Expect(crClient.Delete(ctx, pvc)).To(Succeed()) + }) }) + }) }) } diff --git a/test/e2e/setup_teardown.go b/test/e2e/setup_teardown.go index e20d3e5a6..d9ef44db4 100644 --- a/test/e2e/setup_teardown.go +++ b/test/e2e/setup_teardown.go @@ -18,8 +18,8 @@ package e2e import ( "context" - - "github.com/onsi/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" v1alpha1 "github.com/openshift/lvm-operator/api/v1alpha1" ) @@ -27,54 +27,54 @@ import ( func beforeTestSuiteSetup(ctx context.Context) { if diskInstall { - debug("Creating disk for e2e tests") + By("Creating disk for e2e tests") err := diskSetup(ctx) - gomega.Expect(err).To(gomega.BeNil()) + Expect(err).To(BeNil()) } if lvmOperatorInstall { - debug("BeforeTestSuite: deploying LVM Operator") + By("BeforeTestSuite: deploying LVM Operator") err := deployLVMWithOLM(ctx, lvmCatalogSourceImage, lvmSubscriptionChannel) - gomega.Expect(err).To(gomega.BeNil()) + Expect(err).To(BeNil()) } } func lvmClusterSetup(clusterConfig *v1alpha1.LVMCluster, ctx context.Context) { - debug("BeforeTestSuite: starting LVM Cluster") + By("Starting LVM Cluster") err := startLVMCluster(clusterConfig, ctx) - gomega.Expect(err).To(gomega.BeNil()) + Expect(err).To(BeNil()) } func lvmNamespaceSetup(ctx context.Context) { - debug("BeforeTestSuite: creating Namespace", testNamespace) + By("Creating Namespace " + testNamespace) err := createNamespace(ctx, testNamespace) - gomega.Expect(err).To(gomega.BeNil()) + Expect(err).To(BeNil()) } func lvmNamespaceCleanup(ctx context.Context) { - debug("AfterTestSuite: deleting Namespace", testNamespace) + By("Deleting Namespace " + testNamespace) err := deleteNamespaceAndWait(ctx, testNamespace) - gomega.Expect(err).To(gomega.BeNil()) + Expect(err).To(BeNil()) } func lvmClusterCleanup(clusterConfig *v1alpha1.LVMCluster, ctx context.Context) { - debug("AfterTestSuite: deleting default LVM Cluster") + By("Deleting default LVM Cluster") err := deleteLVMCluster(clusterConfig, ctx) - gomega.Expect(err).To(gomega.BeNil()) + Expect(err).To(BeNil()) } // afterTestSuiteCleanup is the function called to tear down the test environment. func afterTestSuiteCleanup(ctx context.Context) { if lvmOperatorUninstall { - debug("AfterTestSuite: uninstalling LVM Operator") + By("AfterTestSuite: uninstalling LVM Operator") err := uninstallLVM(ctx, lvmCatalogSourceImage, lvmSubscriptionChannel) - gomega.Expect(err).To(gomega.BeNil(), "error uninstalling the LVM Operator: %v", err) + Expect(err).To(BeNil(), "error uninstalling the LVM Operator: %v", err) } if diskInstall { - debug("Cleaning up disk") + By("Cleaning up disk") err := diskRemoval(ctx) - gomega.Expect(err).To(gomega.BeNil()) + Expect(err).To(BeNil()) } } diff --git a/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-device.yaml b/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-device.yaml index f0b120e1d..4e602c8ac 100644 --- a/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-device.yaml +++ b/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-device.yaml @@ -6,6 +6,13 @@ metadata: spec: containers: - name: ubuntu + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - "ALL" image: quay.io/cybozu/ubuntu:20.04 command: ["/usr/local/bin/pause"] volumeDevices: diff --git a/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-mount.yaml b/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-mount.yaml index 8a8402e80..7d1d98784 100644 --- a/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-mount.yaml +++ b/test/e2e/testdata/ephemeral_tests/pod-ephemeral-volume-mount.yaml @@ -6,6 +6,13 @@ metadata: spec: containers: - name: ubuntu + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - "ALL" image: quay.io/cybozu/ubuntu:20.04 command: ["/usr/local/bin/pause"] volumeMounts: diff --git a/test/e2e/testdata/ephemeral_tests/pod-volume-device-template.yaml b/test/e2e/testdata/ephemeral_tests/pod-volume-device-template.yaml index 119fc9a5b..e7fdf5545 100644 --- a/test/e2e/testdata/ephemeral_tests/pod-volume-device-template.yaml +++ b/test/e2e/testdata/ephemeral_tests/pod-volume-device-template.yaml @@ -6,6 +6,13 @@ metadata: spec: containers: - name: ubuntu + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - "ALL" image: quay.io/cybozu/ubuntu:20.04 command: ["/usr/local/bin/pause"] volumeDevices: diff --git a/test/e2e/testdata/ephemeral_tests/pod-volume-mount-template.yaml b/test/e2e/testdata/ephemeral_tests/pod-volume-mount-template.yaml index 81dd5484f..4d463a776 100644 --- a/test/e2e/testdata/ephemeral_tests/pod-volume-mount-template.yaml +++ b/test/e2e/testdata/ephemeral_tests/pod-volume-mount-template.yaml @@ -6,6 +6,13 @@ metadata: spec: containers: - name: nginx + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - "ALL" image: public.ecr.aws/docker/library/nginx:latest volumeMounts: - mountPath: /test1 diff --git a/test/e2e/testdata/pvc_tests/pod-volume-device-template.yaml b/test/e2e/testdata/pvc_tests/pod-volume-device-template.yaml index 119fc9a5b..e7fdf5545 100644 --- a/test/e2e/testdata/pvc_tests/pod-volume-device-template.yaml +++ b/test/e2e/testdata/pvc_tests/pod-volume-device-template.yaml @@ -6,6 +6,13 @@ metadata: spec: containers: - name: ubuntu + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - "ALL" image: quay.io/cybozu/ubuntu:20.04 command: ["/usr/local/bin/pause"] volumeDevices: diff --git a/test/e2e/testdata/pvc_tests/pod-volume-mount-template.yaml b/test/e2e/testdata/pvc_tests/pod-volume-mount-template.yaml index 81dd5484f..4d463a776 100644 --- a/test/e2e/testdata/pvc_tests/pod-volume-mount-template.yaml +++ b/test/e2e/testdata/pvc_tests/pod-volume-mount-template.yaml @@ -6,6 +6,13 @@ metadata: spec: containers: - name: nginx + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - "ALL" image: public.ecr.aws/docker/library/nginx:latest volumeMounts: - mountPath: /test1 diff --git a/test/e2e/validation.go b/test/e2e/validation.go index 38ad943bf..1c6d24694 100644 --- a/test/e2e/validation.go +++ b/test/e2e/validation.go @@ -18,6 +18,9 @@ package e2e import ( "context" + "errors" + "fmt" + "k8s.io/client-go/discovery" "time" . "github.com/onsi/ginkgo/v2" @@ -31,8 +34,8 @@ import ( ) const ( - timeout = time.Minute * 10 - interval = time.Second * 15 + timeout = time.Minute * 2 + interval = time.Second * 3 lvmVolumeGroupName = "vg1" storageClassName = "lvms-vg1" volumeSnapshotClassName = "lvms-vg1" @@ -43,159 +46,109 @@ const ( ) // function to validate LVMVolume group. -func validateLVMvg(ctx context.Context) error { - lvmVG := lvmv1alpha1.LVMVolumeGroup{} - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: lvmVolumeGroupName, Namespace: installNamespace}, &lvmVG) - if err != nil { - debug("Error getting LVMVolumeGroup %s: %s\n", lvmVolumeGroupName, err.Error()) - } - return err == nil - }, timeout, interval).Should(BeTrue()) - - debug("VG found\n") - return nil +func validateLVMvg(ctx context.Context) bool { + By("validating the LVMVolumeGroup") + return Eventually(func(ctx context.Context) error { + return crClient.Get(ctx, types.NamespacedName{Name: lvmVolumeGroupName, Namespace: installNamespace}, &lvmv1alpha1.LVMVolumeGroup{}) + }, timeout, interval).WithContext(ctx).Should(Succeed()) } // function to validate storage class. -func validateStorageClass(ctx context.Context) error { - sc := storagev1.StorageClass{} - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &sc) - if err != nil { - debug("Error getting StorageClass %s: %s\n", storageClassName, err.Error()) - } - return err == nil - }, timeout, interval).Should(BeTrue()) - - debug("SC found\n") - return nil +func validateStorageClass(ctx context.Context) bool { + By("validating the StorageClass") + return Eventually(func() error { + return crClient.Get(ctx, types.NamespacedName{Name: storageClassName, Namespace: installNamespace}, &storagev1.StorageClass{}) + }, timeout, interval).WithContext(ctx).Should(Succeed()) } // function to validate volume snapshot class. -func validateVolumeSnapshotClass(ctx context.Context) error { - vsc := snapapi.VolumeSnapshotClass{} - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: volumeSnapshotClassName}, &vsc) - if err != nil { - debug("Error getting VolumeSnapshotClass %s: %s\n", volumeSnapshotClassName, err.Error()) +func validateVolumeSnapshotClass(ctx context.Context) bool { + By("validating the VolumeSnapshotClass") + return Eventually(func(ctx context.Context) error { + err := crClient.Get(ctx, types.NamespacedName{Name: volumeSnapshotClassName}, &snapapi.VolumeSnapshotClass{}) + if discovery.IsGroupDiscoveryFailedError(errors.Unwrap(err)) { + By("VolumeSnapshotClass is ignored since VolumeSnapshotClasses are not supported in the given Cluster") + return nil } - return err == nil - }, timeout, interval).Should(BeTrue()) - - debug("VolumeSnapshotClass found\n") - return nil + return err + }, timeout, interval).WithContext(ctx).Should(Succeed()) } // function to validate CSI Driver. -func validateCSIDriver(ctx context.Context) error { - cd := storagev1.CSIDriver{} - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: csiDriverName, Namespace: installNamespace}, &cd) - if err != nil { - debug("Error getting CSIDriver %s: %s\n", csiDriverName, err.Error()) - } - return err == nil - }, timeout, interval).Should(BeTrue()) - - debug("CSIDriver found\n") - return nil +func validateCSIDriver(ctx context.Context) bool { + By("validating the CSIDriver") + return Eventually(func(ctx context.Context) error { + return crClient.Get(ctx, types.NamespacedName{Name: csiDriverName, Namespace: installNamespace}, &storagev1.CSIDriver{}) + }, timeout, interval).WithContext(ctx).Should(Succeed()) } // function to validate TopoLVM node. -func validateTopolvmNode(ctx context.Context) error { - ds := appsv1.DaemonSet{} - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: topolvmNodeDaemonSetName, Namespace: installNamespace}, &ds) - if err != nil { - debug("Error getting TopoLVM node daemonset %s: %s\n", topolvmNodeDaemonSetName, err.Error()) - } - return err == nil - }, timeout, interval).Should(BeTrue()) - debug("TopoLVM node daemonset found\n") - - // checking for the ready status - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: topolvmNodeDaemonSetName, Namespace: installNamespace}, &ds) - if err != nil { - debug("Error getting TopoLVM node daemonset %s: %s\n", topolvmNodeDaemonSetName, err.Error()) - } - return ds.Status.DesiredNumberScheduled == ds.Status.NumberReady - }, timeout, interval).Should(BeTrue()) - debug("TopoLVM node pods are ready\n") - - return nil +func validateTopolvmNode(ctx context.Context) bool { + By("validating the TopoLVM Node DaemonSet") + return validateDaemonSet(ctx, types.NamespacedName{Name: topolvmNodeDaemonSetName, Namespace: installNamespace}) } // function to validate vg manager resource. -func validateVGManager(ctx context.Context) error { - ds := appsv1.DaemonSet{} - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: vgManagerDaemonsetName, Namespace: installNamespace}, &ds) - if err != nil { - debug("Error getting vg manager daemonset %s: %s\n", vgManagerDaemonsetName, err.Error()) - } - return err == nil - }, timeout, interval).Should(BeTrue()) - debug("VG manager daemonset found\n") - - return nil +func validateVGManager(ctx context.Context) bool { + By("validating the vg-manager DaemonSet") + return validateDaemonSet(ctx, types.NamespacedName{Name: vgManagerDaemonsetName, Namespace: installNamespace}) } // function to validate TopoLVM Deployment. -func validateTopolvmController(ctx context.Context) error { - dep := appsv1.Deployment{} - - Eventually(func() bool { - err := crClient.Get(ctx, types.NamespacedName{Name: topolvmCtrlDeploymentName, Namespace: installNamespace}, &dep) - if err != nil { - debug("Error getting TopoLVM controller deployment %s: %s\n", topolvmCtrlDeploymentName, err.Error()) +func validateTopolvmController(ctx context.Context) bool { + By("validating the TopoLVM controller deployment") + name := types.NamespacedName{Name: topolvmCtrlDeploymentName, Namespace: installNamespace} + return Eventually(func(ctx context.Context) error { + deploy := &appsv1.Deployment{} + if err := crClient.Get(ctx, name, deploy); err != nil { + return err } - return err == nil - }, timeout, interval).Should(BeTrue()) + isReady := deploy.Spec.Replicas != nil && *deploy.Spec.Replicas == deploy.Status.ReadyReplicas + if !isReady { + return fmt.Errorf("the Deployment %s is not considered ready", name) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) +} - debug("TopoLVM controller deployment found\n") - return nil +func validateDaemonSet(ctx context.Context, name types.NamespacedName) bool { + return Eventually(func(ctx context.Context) error { + ds := &appsv1.DaemonSet{} + if err := crClient.Get(ctx, name, ds); err != nil { + return err + } + isReady := ds.Status.DesiredNumberScheduled == ds.Status.NumberReady + if !isReady { + return fmt.Errorf("the DaemonSet %s is not considered ready", name) + } + return nil + }, timeout, interval).WithContext(ctx).Should(Succeed()) } // Validate all the resources created by LVMO. func validateResources() { - - var ctx = context.Background() Describe("Validate LVMCluster reconciliation", func() { - It("Should check that LVMO resources have been created", func() { + It("Should check that LVMO resources have been created", func(ctx SpecContext) { By("Checking that CSIDriver has been created") - err := validateCSIDriver(ctx) - Expect(err).To(BeNil()) + Expect(validateCSIDriver(ctx)).To(BeTrue()) By("Checking that the topolvm-controller deployment has been created") - err = validateTopolvmController(ctx) - Expect(err).To(BeNil()) + Expect(validateTopolvmController(ctx)).To(BeTrue()) By("Checking that the vg-manager daemonset has been created") - err = validateVGManager(ctx) - Expect(err).To(BeNil()) + Expect(validateVGManager(ctx)).To(BeTrue()) By("Checking that the LVMVolumeGroup has been created") - err = validateLVMvg(ctx) - Expect(err).To(BeNil()) + Expect(validateLVMvg(ctx)).To(BeTrue()) By("Checking that the topolvm-node daemonset has been created") - err = validateTopolvmNode(ctx) - Expect(err).To(BeNil()) + Expect(validateTopolvmNode(ctx)).To(BeTrue()) By("Checking that the StorageClass has been created") - err = validateStorageClass(ctx) - Expect(err).To(BeNil()) + Expect(validateStorageClass(ctx)).To(BeTrue()) By("Checking that the VolumeSnapshotClass has been created") - err = validateVolumeSnapshotClass(ctx) - Expect(err).To(BeNil()) + Expect(validateVolumeSnapshotClass(ctx)).To(BeTrue()) }) }) }