Skip to content

Commit

Permalink
add e2e test for volume resizing
Browse files Browse the repository at this point in the history
  • Loading branch information
AndyXiangLi committed Jan 30, 2021
1 parent 511dd1c commit c5eba4f
Show file tree
Hide file tree
Showing 8 changed files with 139 additions and 14 deletions.
4 changes: 3 additions & 1 deletion charts/aws-ebs-csi-driver/templates/clusterrole-resizer.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,7 @@ rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]

- apiGroups: [""]
resources: ["pods"]
verbs: ["get", "list", "watch"]
{{- end}}
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,6 @@ rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [ "" ]
resources: [ "pods" ]
verbs: [ "get", "list", "watch" ]
16 changes: 9 additions & 7 deletions tests/e2e/driver/driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ type PVTestDriver interface {
// DynamicPVTestDriver represents an interface for a CSI driver that supports DynamicPV
type DynamicPVTestDriver interface {
// GetDynamicProvisionStorageClass returns a StorageClass dynamic provision Persistent Volume
GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, volumeExpansion *bool, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass
}

// PreProvisionedVolumeTestDriver represents an interface for a CSI driver that supports pre-provisioned volume
Expand All @@ -54,6 +54,7 @@ func getStorageClass(
parameters map[string]string,
mountOptions []string,
reclaimPolicy *v1.PersistentVolumeReclaimPolicy,
volumeExpansion *bool,
bindingMode *storagev1.VolumeBindingMode,
allowedTopologies []v1.TopologySelectorTerm,
) *storagev1.StorageClass {
Expand All @@ -69,12 +70,13 @@ func getStorageClass(
ObjectMeta: metav1.ObjectMeta{
GenerateName: generateName,
},
Provisioner: provisioner,
Parameters: parameters,
MountOptions: mountOptions,
ReclaimPolicy: reclaimPolicy,
VolumeBindingMode: bindingMode,
AllowedTopologies: allowedTopologies,
Provisioner: provisioner,
Parameters: parameters,
MountOptions: mountOptions,
ReclaimPolicy: reclaimPolicy,
VolumeBindingMode: bindingMode,
AllowedTopologies: allowedTopologies,
AllowVolumeExpansion: volumeExpansion,
}
}

Expand Down
5 changes: 3 additions & 2 deletions tests/e2e/driver/ebs_csi_driver.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,11 @@ func InitEbsCSIDriver() PVTestDriver {
}
}

func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]string, mountOptions []string, reclaimPolicy *v1.PersistentVolumeReclaimPolicy, volumeExpansion *bool, bindingMode *storagev1.VolumeBindingMode, allowedTopologyValues []string, namespace string) *storagev1.StorageClass {
provisioner := d.driverName
generateName := fmt.Sprintf("%s-%s-dynamic-sc-", namespace, provisioner)
allowedTopologies := []v1.TopologySelectorTerm{}

if len(allowedTopologyValues) > 0 {
allowedTopologies = []v1.TopologySelectorTerm{
{
Expand All @@ -57,7 +58,7 @@ func (d *ebsCSIDriver) GetDynamicProvisionStorageClass(parameters map[string]str
},
}
}
return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, bindingMode, allowedTopologies)
return getStorageClass(generateName, provisioner, parameters, mountOptions, reclaimPolicy, volumeExpansion, bindingMode, allowedTopologies)
}

func (d *ebsCSIDriver) GetVolumeSnapshotClass(namespace string) *v1beta1.VolumeSnapshotClass {
Expand Down
24 changes: 24 additions & 0 deletions tests/e2e/dynamic_provisioning.go
Original file line number Diff line number Diff line change
Expand Up @@ -404,6 +404,30 @@ var _ = Describe("[ebs-csi-e2e] [single-az] Dynamic Provisioning", func() {
}
test.Run(cs, ns)
})

It("should create a volume on demand and resize it ", func() {
allowVolumeExpansion := true
pod := testsuites.PodDetails{
Cmd: "echo 'hello world' >> /mnt/test-1/data && grep 'hello world' /mnt/test-1/data && sync",
Volumes: []testsuites.VolumeDetails{
{
VolumeType: awscloud.VolumeTypeGP2,
FSType: ebscsidriver.FSTypeExt4,
ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2),
VolumeMount: testsuites.VolumeMountDetails{
NameGenerate: "test-volume-",
MountPathGenerate: "/mnt/test-",
},
AllowVolumeExpansion: &allowVolumeExpansion,
},
},
}
test := testsuites.DynamicallyProvisionedResizeVolumeTest{
CSIDriver: ebsDriver,
Pod: pod,
}
test.Run(cs, ns)
})
})

var _ = Describe("[ebs-csi-e2e] [single-az] Snapshot", func() {
Expand Down
3 changes: 1 addition & 2 deletions tests/e2e/pre_provsioning.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package e2e
import (
"context"
"fmt"
ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
k8srestclient "k8s.io/client-go/rest"
"math/rand"
"os"
Expand All @@ -29,8 +30,6 @@ import (
v1 "k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"

ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
)

const (
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testsuites

import (
"fmt"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util"
"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/kubernetes/test/e2e/framework"
"time"

. "github.com/onsi/ginkgo"
"k8s.io/api/core/v1"
clientset "k8s.io/client-go/kubernetes"
)

// DynamicallyProvisionedResizeVolumeTest will provision required StorageClass(es), PVC(s) and Pod(s)
// Waiting for the PV provisioner to create a new PV
// Update pvc storage size
// Waiting for new PVC and PV to be ready
// And finally attach pvc to the pod and wait for pod to be ready.
type DynamicallyProvisionedResizeVolumeTest struct {
CSIDriver driver.DynamicPVTestDriver
Pod PodDetails
}

func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface, namespace *v1.Namespace) {
volume := t.Pod.Volumes[0]
tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
defer tpvc.Cleanup()

pvcName := tpvc.persistentVolumeClaim.Name
pvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(pvcName, metav1.GetOptions{})
By(fmt.Sprintf("Get pvc name: %v", pvc.Name))
originalSize := pvc.Spec.Resources.Requests["storage"]
delta := resource.Quantity{}
delta.Set(util.GiBToBytes(1))
originalSize.Add(delta)
pvc.Spec.Resources.Requests["storage"] = originalSize

By("resizing the pvc")
updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(pvc)
if err != nil {
framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err))
}
updatedSize := updatedPvc.Spec.Resources.Requests["storage"]

By("checking the resizing PV result")
error := WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
framework.ExpectNoError(error)

By("Validate volume can be attached")
tpod := NewTestPod(client, namespace, t.Pod.Cmd)

tpod.SetupVolume(tpvc.persistentVolumeClaim, volume.VolumeMount.NameGenerate+"1", volume.VolumeMount.MountPathGenerate+"1", volume.VolumeMount.ReadOnly)

By("deploying the pod")
tpod.Create()
By("checking that the pods is running")
tpod.WaitForSuccess()

defer tpod.Cleanup()

}

// WaitForPvToResize waiting for pvc size to be resized to desired size
func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, desiredSize resource.Quantity, timeout time.Duration, interval time.Duration) error {
By(fmt.Sprintf("Waiting up to %v for pv in namespace %q to be complete", timeout, ns.Name))
for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) {
newPv, _ := c.CoreV1().PersistentVolumes().Get(pvName, metav1.GetOptions{})
newPvSize := newPv.Spec.Capacity["storage"]
if desiredSize.Equal(newPvSize) {
By(fmt.Sprintf("Pv size is updated to %v", newPvSize.String()))
return nil
}
}
return fmt.Errorf("Gave up after waiting %v for pv %q to complete resizing", timeout, pvName)
}
6 changes: 4 additions & 2 deletions tests/e2e/testsuites/specs.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ type VolumeDetails struct {
MountOptions []string
ClaimSize string
ReclaimPolicy *v1.PersistentVolumeReclaimPolicy
AllowVolumeExpansion *bool
VolumeBindingMode *storagev1.VolumeBindingMode
AllowedTopologyValues []string
VolumeMode VolumeMode
Expand Down Expand Up @@ -119,7 +120,8 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
cleanupFuncs := make([]func(), 0)
volume := pod.Volumes[0]
By("setting up the StorageClass")
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)

storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
tsc := NewTestStorageClass(client, namespace, storageClass)
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
Expand All @@ -139,7 +141,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPersistentVolumeClaim, []func()) {
cleanupFuncs := make([]func(), 0)
By("setting up the StorageClass")
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
tsc := NewTestStorageClass(client, namespace, storageClass)
createdStorageClass := tsc.Create()
cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
Expand Down

0 comments on commit c5eba4f

Please sign in to comment.