From da63679b4c5109cd269ec2f7e0d226134e0c6063 Mon Sep 17 00:00:00 2001 From: Xiang Li Date: Wed, 27 Jan 2021 10:15:46 -0800 Subject: [PATCH] add e2e test for volume resizing --- tests/e2e/driver/driver.go | 16 ++-- tests/e2e/driver/ebs_csi_driver.go | 5 +- tests/e2e/dynamic_provisioning.go | 24 +++++ tests/e2e/pre_provsioning.go | 3 +- ...ically_provisioned_resize_volume_tester.go | 92 +++++++++++++++++++ tests/e2e/testsuites/specs.go | 6 +- 6 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go diff --git a/tests/e2e/driver/driver.go b/tests/e2e/driver/driver.go index 4b54e6912c..853bda4eb1 100644 --- a/tests/e2e/driver/driver.go +++ b/tests/e2e/driver/driver.go @@ -35,7 +35,7 @@ type PVTestDriver interface { // DynamicPVTestDriver represents an interface for a CSI driver that supports DynamicPV type DynamicPVTestDriver interface { // GetDynamicProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume - GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass + GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, volumeExpansion *bool, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass } // PreProvisionedVolumeTestDriver represents an interface for a CSI driver that supports pre-provisioned volume @@ -54,6 +54,7 @@ func getStorageClass( parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, + volumeExpansion *bool, bindingMode *storagev1.VolumeBindingMode, allowedTopologies []v1.TopologySelectorTerm, ) *storagev1.StorageClass { @@ -69,12 +70,13 @@ func getStorageClass( ObjectMeta: metav1.ObjectMeta{ GenerateName: generateName, }, - Provisioner: provisioner, - Parameters: parameters, - MountOptions: mountOptions, - ReclaimPolicy: reclaimPolicy, - VolumeBindingMode: bindingMode, - AllowedTopologies: allowedTopologies, + Provisioner: provisioner, + Parameters: parameters, + MountOptions: mountOptions, + ReclaimPolicy: reclaimPolicy, + VolumeBindingMode: bindingMode, + AllowedTopologies: allowedTopologies, + AllowVolumeExpansion: volumeExpansion, } } diff --git a/tests/e2e/driver/ebs_csi_driver.go b/tests/e2e/driver/ebs_csi_driver.go index 94260f1309..9a72392ed4 100644 --- a/tests/e2e/driver/ebs_csi_driver.go +++ b/tests/e2e/driver/ebs_csi_driver.go @@ -41,10 +41,11 @@ func InitEbsCSIDriver() PVTestDriver { } } -func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass { +func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, volumeExpansion *bool, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass { provisioner := d.driverName generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner) allowedTopologies := []v1.TopologySelectorTerm{} + if len(allowedTopologyValues) > 0 { allowedTopologies = []v1.TopologySelectorTerm{ { @@ -57,7 +58,7 @@ func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]str }, } } - return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, allowedTopologies) + return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, volumeExpansion, bindingMode, allowedTopologies) } func (d *ebsCSIDriver) GetVolumeSnapshotClass(namespace string) *v1beta1.VolumeSnapshotClass { diff --git a/tests/e2e/dynamic_provisioning.go b/tests/e2e/dynamic_provisioning.go index 104021d3ff..a5ab32d6b6 100644 --- a/tests/e2e/dynamic_provisioning.go +++ b/tests/e2e/dynamic_provisioning.go @@ -404,6 +404,30 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() { } test.Run(cs, ns) }) + + It("should create a volume on demand and resize it ", func() { + allowVolumeExpansion := true + pod := testsuites.PodDetails{ + Cmd: "echo 'hello world' >> /mnt/test-1/data && grep 'hello world' /mnt/test-1/data && sync", + Volumes: []testsuites.VolumeDetails{ + { + VolumeType: awscloud.VolumeTypeGP2, + FSType: ebscsidriver.FSTypeExt4, + ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2), + VolumeMount: testsuites.VolumeMountDetails{ + NameGenerate: "test-volume-", + MountPathGenerate: "/mnt/test-", + }, + AllowVolumeExpansion: &allowVolumeExpansion, + }, + }, + } + test := testsuites.DynamicallyProvisionedResizeVolumeTest{ + CSIDriver: ebsDriver, + Pod: pod, + } + test.Run(cs, ns) + }) }) var _ = Describe("[ebs-csi-e2e] [single-az] Snapshot", func() { diff --git a/tests/e2e/pre_provsioning.go b/tests/e2e/pre_provsioning.go index 06ef2d86ab..7612279466 100644 --- a/tests/e2e/pre_provsioning.go +++ b/tests/e2e/pre_provsioning.go @@ -17,6 +17,7 @@ package e2e import ( "context" "fmt" + ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" k8srestclient "k8s.io/client-go/rest" "math/rand" "os" @@ -29,8 +30,6 @@ import ( v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/test/e2e/framework" - - ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" ) const ( diff --git a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go new file mode 100644 index 0000000000..5f1dc0f2c6 --- /dev/null +++ b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go @@ -0,0 +1,92 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "fmt" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/kubernetes/test/e2e/framework" + "time" + + . "github.com/onsi/ginkgo" + "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" +) + +// DynamicallyProvisionedResizeVolumeTest will provision required StorageClass(es), PVC(s) and Pod(s) +// Waiting for the PV provisioner to create a new PV +// Update pvc storage size +// Waiting for new PVC and PV to be ready +// And finally attach pvc to the pod and wait for pod to be ready. +type DynamicallyProvisionedResizeVolumeTest struct { + CSIDriver driver.DynamicPVTestDriver + Pod PodDetails +} + +func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface, namespace *v1.Namespace) { + volume := t.Pod.Volumes[0] + tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver) + defer tpvc.Cleanup() + + pvcName := tpvc.persistentVolumeClaim.Name + pvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(pvcName, metav1.GetOptions{}) + By(fmt.Sprintf("Get pvc name: %v", pvc.Name)) + originalSize := pvc.Spec.Resources.Requests["storage"] + delta := resource.Quantity{} + delta.Set(util.GiBToBytes(1)) + originalSize.Add(delta) + pvc.Spec.Resources.Requests["storage"] = originalSize + + By("resizing the pvc") + updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(pvc) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err)) + } + updatedSize := updatedPvc.Spec.Resources.Requests["storage"] + + By("checking the resizing PV result") + error := WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1 * time.Minute, 5*time.Second) + framework.ExpectNoError(error) + + By("Validate volume can be attached") + tpod := NewTestPod(client, namespace, t.Pod.Cmd) + + tpod.SetupVolume(tpvc.persistentVolumeClaim, volume.VolumeMount.NameGenerate+"1", volume.VolumeMount.MountPathGenerate+"1", volume.VolumeMount.ReadOnly) + + By("deploying the pod") + tpod.Create() + By("checking that the pods is running") + tpod.WaitForSuccess() + + defer tpod.Cleanup() + +} + +// WaitForPvToResize waiting for pvc size to be resized to desired size +func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, desiredSize resource.Quantity, timeout time.Duration, interval time.Duration) error { + By(fmt.Sprintf("Waiting up to %v for pv in namespace %q to be complete", timeout, ns.Name)) + for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) { + newPv, _ := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{}) + newPvSize := newPv.Spec.Capacity["storage"] + if desiredSize.Equal(newPvSize) { + By(fmt.Sprintf("Pv size is updated to %v", newPvSize.String())) + return nil + } + } + return fmt.Errorf("Gave up after waiting %v for pvc %q to complete resizing", timeout, pvName) +} \ No newline at end of file diff --git a/tests/e2e/testsuites/specs.go b/tests/e2e/testsuites/specs.go index cbe30c507f..6c43d6cf01 100644 --- a/tests/e2e/testsuites/specs.go +++ b/tests/e2e/testsuites/specs.go @@ -39,6 +39,7 @@ type VolumeDetails struct { MountOptions []string ClaimSize string ReclaimPolicy *v1.PersistentVolumeReclaimPolicy + AllowVolumeExpansion *bool VolumeBindingMode *storagev1.VolumeBindingMode AllowedTopologyValues []string VolumeMode VolumeMode @@ -119,7 +120,8 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1 cleanupFuncs := make([]func(), 0) volume := pod.Volumes[0] By("setting up the StorageClass") - storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) + + storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) tsc := NewTestStorageClass(client, namespace, storageClass) createdStorageClass := tsc.Create() cleanupFuncs = append(cleanupFuncs, tsc.Cleanup) @@ -139,7 +141,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1 func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPersistentVolumeClaim, []func()) { cleanupFuncs := make([]func(), 0) By("setting up the StorageClass") - storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) + storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) tsc := NewTestStorageClass(client, namespace, storageClass) createdStorageClass := tsc.Create() cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)