[Hold] Add format option e2e tests

tests/e2e/driver/ebs_csi_driver.go

@@ -103,7 +103,7 @@ func (d *ebsCSIDriver) GetPersistentVolume(volumeID string, fsType string, size
 }
 
 // GetParameters returns the parameters specific for this driver
-func GetParameters(volumeType string, fsType string, encrypted bool) map[string]string {
+func GetParameters(volumeType string, fsType string, encrypted bool, additionalParameters map[string]string) map[string]string {
 	parameters := map[string]string{
 		"type":                      volumeType,
 		"csi.storage.k8s.io/fstype": fsType,
@@ -120,6 +120,10 @@ func GetParameters(volumeType string, fsType string, encrypted bool) map[string]
 	if encrypted {
 		parameters[ebscsidriver.EncryptedKey] = True
 	}
+	for k, v := range additionalParameters {
+		parameters[k] = v
+	}
+
 	return parameters
 }
 

tests/e2e/format_options.go

@@ -0,0 +1,197 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
+	ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
+	"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
+	"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites"
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega/format"
+	v1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	admissionapi "k8s.io/pod-security-admission/api"
+	"regexp"
+	"strconv"
+)
+
+const (
+	initialVolumeSizeGi     = 2
+	volumeSizeIncreaseAmtGi = 2
+
+	blockSizeTestValue      = "1024"
+	inodeSizeTestValue      = "512"
+	bytesPerInodeTestValue  = "8192"
+	numberOfInodesTestValue = "200192"
+)
+
+type formatOptionTestCase struct {
+	createVolumeParameterKey        string
+	createVolumeParameterValue      string
+	expectedFilesystemInfoParamName string
+	expectedFilesystemInfoParamVal  string
+}
+
+// TODO is this a clean place for this? Or should we follow standard of other e2e tests with tester files in e2e/testsuites?
+var (
+	// TODO having this here is code smell. Hardcode? Test case from original inode PR #1661 https://github.com/kubernetes-sigs/aws-ebs-csi-driver/pull/1661
+	expectedBytesPerInodeTestResult = strconv.Itoa(131072 * initialVolumeSizeGi)
+
+	formatOptionTests = []formatOptionTestCase{
+		{
+			createVolumeParameterKey:        ebscsidriver.BlockSizeKey,
+			createVolumeParameterValue:      blockSizeTestValue,
+			expectedFilesystemInfoParamName: "Block size",
+			expectedFilesystemInfoParamVal:  blockSizeTestValue,
+		},
+		{
+			createVolumeParameterKey:        ebscsidriver.INodeSizeKey,
+			createVolumeParameterValue:      inodeSizeTestValue,
+			expectedFilesystemInfoParamName: "Inode size",
+			expectedFilesystemInfoParamVal:  inodeSizeTestValue},
+		{
+			createVolumeParameterKey:        ebscsidriver.BytesPerINodeKey,
+			createVolumeParameterValue:      bytesPerInodeTestValue,
+			expectedFilesystemInfoParamName: "Inode count",
+			expectedFilesystemInfoParamVal:  expectedBytesPerInodeTestResult,
+		},
+		{
+			createVolumeParameterKey:        ebscsidriver.NumberOfINodesKey,
+			createVolumeParameterValue:      numberOfInodesTestValue,
+			expectedFilesystemInfoParamName: "Inode count",
+			expectedFilesystemInfoParamVal:  numberOfInodesTestValue},
+	}
+)
+
+var _ = Describe("[ebs-csi-e2e] [format-options] Formatting a volume", func() {
+	f := framework.NewDefaultFramework("ebs")
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged // TODO Maybe don't need this if Connor big brain pulls thru
+
+	var (
+		cs        clientset.Interface
+		ns        *v1.Namespace
+		ebsDriver driver.PVTestDriver
+
+		testedFsTypes = []string{ebscsidriver.FSTypeExt4} // TODO Is this right place for this?
+
+		volumeMountPath     = "/mnt/test-format-option"                                                                                            // TODO maybe keep this as mnt/test-1, and refactor to be `DefaultMountPath` globally in testsuites.
+		podCmdGetFsInfo     = fmt.Sprintf("tune2fs -l $(df -k '%s'| tail -1 | awk '{ print $1 }')", volumeMountPath)                               // Gets the filesystem info for the mounted volume
+		podCmdWriteToVolume = fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath) // TODO Debt: All the dynamic provisioning tests use this same cmd. Should we refactor out into exported constant?
+	)
+
+	BeforeEach(func() {
+		cs = f.ClientSet
+		ns = f.Namespace
+		ebsDriver = driver.InitEbsCSIDriver()
+	})
+
+	for _, fsType := range testedFsTypes {
+		Context(fmt.Sprintf("with an %s filesystem", fsType), func() {
+			// TODO: is t clear? Or should it be 'formatOptionTestCaseValues'
+			for _, t := range formatOptionTests {
+				if fsTypeDoesNotSupportFormatOptionParameter(fsType, t.createVolumeParameterKey) {
+					continue
+				}
+
+				Context(fmt.Sprintf("with a custom %s parameter", t.createVolumeParameterKey), func() {
+					It("successfully mounts and is resizable", func() {
+						By("setting up pvc")
+						volumeDetails := createFormatOptionVolumeDetails(fsType, volumeMountPath, t)
+						testPvc, _ := volumeDetails.SetupDynamicPersistentVolumeClaim(cs, ns, ebsDriver)
+						defer testPvc.Cleanup()
+
+						By("deploying pod with custom format option")
+						getFsInfoTestPod := createPodWithVolume(cs, ns, podCmdGetFsInfo, testPvc, volumeDetails)
+						defer getFsInfoTestPod.Cleanup()
+						getFsInfoTestPod.WaitForSuccess() // TODO e2e test implementation defaults to a 15 min wait instead of 5 min one... Is that fine or refactor worthy?
+
+						By("confirming custom format option was applied")
+						fsInfoSearchRegexp := fmt.Sprintf(`%s:\s+%s`, t.expectedFilesystemInfoParamName, t.expectedFilesystemInfoParamVal)
+						if isFormatOptionApplied := FindRegexpInPodLogs(fsInfoSearchRegexp, getFsInfoTestPod); !isFormatOptionApplied {
+							framework.Failf("Did not find expected %s value of %s in filesystem info", t.expectedFilesystemInfoParamName, t.expectedFilesystemInfoParamVal)
+						}
+
+						By("testing that pvc is able to be resized")
+						testsuites.ResizePvc(cs, ns, testPvc, volumeSizeIncreaseAmtGi)
+
+						By("validating resized pvc by deploying new pod")
+						resizeTestPod := createPodWithVolume(cs, ns, podCmdWriteToVolume, testPvc, volumeDetails)
+						defer resizeTestPod.Cleanup()
+
+						By("confirming new pod can write to resized volume")
+						resizeTestPod.WaitForSuccess()
+					})
+				})
+			}
+		})
+	}
+})
+
+func fsTypeDoesNotSupportFormatOptionParameter(fsType string, createVolumeParameterKey string) bool {
+	_, paramNotSupported := ebscsidriver.FileSystemConfigs[fsType].NotSupportedParams[createVolumeParameterKey]
+	return paramNotSupported
+}
+
+// TODO should we improve this across e2e tests via builder design pattern? Or is that not go-like?
+func createFormatOptionVolumeDetails(fsType string, volumeMountPath string, t formatOptionTestCase) *testsuites.VolumeDetails {
+	allowVolumeExpansion := true
+
+	volume := testsuites.VolumeDetails{
+		VolumeType:   awscloud.VolumeTypeGP2,
+		FSType:       fsType,
+		MountOptions: []string{"rw"},
+		ClaimSize:    fmt.Sprintf("%vGi", initialVolumeSizeGi),
+		VolumeMount: testsuites.VolumeMountDetails{
+			NameGenerate:      "test-volume-format-option",
+			MountPathGenerate: volumeMountPath,
+		},
+		AllowVolumeExpansion: &allowVolumeExpansion,
+		AdditionalParameters: map[string]string{
+			t.createVolumeParameterKey: t.createVolumeParameterValue,
+		},
+	}
+
+	return &volume
+}
+
+// TODO putting this in function may be overkill? In an ideal world we refactor out testsuites.TestEverything objects so testPod.SetupVolume isn't gross.
+func createPodWithVolume(client clientset.Interface, namespace *v1.Namespace, cmd string, testPvc *testsuites.TestPersistentVolumeClaim, volumeDetails *testsuites.VolumeDetails) *testsuites.TestPod {
+	testPod := testsuites.NewTestPod(client, namespace, cmd)
+
+	// TODO Will Refactor in PR 2
+	pvc := testPvc.GetPvc()
+	testPod.SetupVolume(pvc, volumeDetails.VolumeMount.NameGenerate, volumeDetails.VolumeMount.MountPathGenerate, volumeDetails.VolumeMount.ReadOnly)
+
+	testPod.Create()
+
+	return testPod
+}
+
+// TODO: Maybe should use something other than Find(), but *shrug*
+// TODO  I should move this to testsuites.go, yes?
+// TODO Should I instead use RunHostCmd or LookForString from https://github.com/kubernetes/kubernetes/blob/master/test/e2e/framework/pod/output/output.go ?
+
+// FindRegexpInPodLogs searches given testPod's logs for a given regular expression. Returns `true` if found.
+func FindRegexpInPodLogs(regexpPattern string, testPod *testsuites.TestPod) bool {
+	podLogs, err := testPod.Logs()
+	framework.ExpectNoError(err, "Tried getting logs for pod %s", format.Object(testPod, 2))
+
+	var expectedLine = regexp.MustCompile(regexpPattern)
+	res := expectedLine.Find(podLogs)
+	return res != nil
+}

tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go

@@ -45,25 +45,7 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface,
 	tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver)
 	defer tpvc.Cleanup()
 
-	pvcName := tpvc.persistentVolumeClaim.Name
-	pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), pvcName, metav1.GetOptions{})
-	By(fmt.Sprintf("Get pvc name: %v", pvc.Name))
-	originalSize := pvc.Spec.Resources.Requests["storage"]
-	delta := resource.Quantity{}
-	delta.Set(util.GiBToBytes(1))
-	originalSize.Add(delta)
-	pvc.Spec.Resources.Requests["storage"] = originalSize
-
-	By("resizing the pvc")
-	updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
-	if err != nil {
-		framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err))
-	}
-	updatedSize := updatedPvc.Spec.Resources.Requests["storage"]
-
-	By("checking the resizing PV result")
-	error := WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
-	framework.ExpectNoError(error)
+	ResizePvc(client, namespace, tpvc, 1)
 
 	By("Validate volume can be attached")
 	tpod := NewTestPod(client, namespace, t.Pod.Cmd)
@@ -92,3 +74,30 @@ func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, d
 	}
 	return fmt.Errorf("Gave up after waiting %v for pv %q to complete resizing", timeout, pvName)
 }
+
+// TODO move to testsuites.go?
+
+// ResizePvc increases size of given persistent volume claim by `sizeIncreaseGi` Gigabytes
+func ResizePvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedPvc *v1.PersistentVolumeClaim, updatedSize resource.Quantity) {
+	By(fmt.Sprintf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name))
+	pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{})
+
+	originalSize := pvc.Spec.Resources.Requests["storage"]
+	delta := resource.Quantity{}
+	delta.Set(util.GiBToBytes(sizeIncreaseGi))
+	originalSize.Add(delta)
+	pvc.Spec.Resources.Requests["storage"] = originalSize
+
+	By("resizing the pvc")
+	updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{})
+	if err != nil {
+		framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err))
+	}
+	updatedSize = updatedPvc.Spec.Resources.Requests["storage"]
+
+	By("checking the resizing PV result")
+	err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second)
+	framework.ExpectNoError(err)
+
+	return
+}

tests/e2e/testsuites/specs.go

@@ -45,10 +45,9 @@ type VolumeDetails struct {
 	VolumeMode            VolumeMode
 	VolumeMount           VolumeMountDetails
 	VolumeDevice          VolumeDeviceDetails
-	// Optional, used with pre-provisioned volumes
-	VolumeID string
-	// Optional, used with PVCs created from snapshots
-	DataSource *DataSource
+	VolumeID              string            // Optional, used with pre-provisioned volumes
+	DataSource            *DataSource       // Optional, used with PVCs created from snapshots
+	AdditionalParameters  map[string]string // Optional, used when testing formatting options
 }
 
 type VolumeMode int
@@ -121,7 +120,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
 	volume := pod.Volumes[0]
 	By("setting up the StorageClass")
 
-	storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
+	storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
 	tsc := NewTestStorageClass(client, namespace, storageClass)
 	createdStorageClass := tsc.Create()
 	cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)
@@ -141,7 +140,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1
 func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPersistentVolumeClaim, []func()) {
 	cleanupFuncs := make([]func(), 0)
 	By("setting up the StorageClass")
-	storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
+	storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name)
 	tsc := NewTestStorageClass(client, namespace, storageClass)
 	createdStorageClass := tsc.Create()
 	cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)

tests/e2e/testsuites/testsuites.go

@@ -333,6 +333,11 @@ func (t *TestPersistentVolumeClaim) Create() {
 	framework.ExpectNoError(err)
 }
 
+// TODO Refactor and remove in PR 2
+func (t *TestPersistentVolumeClaim) GetPvc() *v1.PersistentVolumeClaim {
+	return t.persistentVolumeClaim
+}
+
 func (t *TestPersistentVolumeClaim) ValidateProvisionedPersistentVolume() {
 	var err error
 
@@ -631,10 +636,14 @@ func NewTestPod(c clientset.Interface, ns *v1.Namespace, command string) *TestPo
 				Containers: []v1.Container{
 					{
 						Name:         "volume-tester",
-						Image:        imageutils.GetE2EImage(imageutils.BusyBox),
+						Image:        "docker.io/ubuntu", // TODO can be refactored out, waiting to see if Connor can big brain way to use busybox
 						Command:      []string{"/bin/sh"},
 						Args:         []string{"-c", command},
 						VolumeMounts: make([]v1.VolumeMount, 0),
+						// TODO Should be refactored out, waiting to see if Connor can big brain a non-privileged way to get fs info
+						SecurityContext: &v1.SecurityContext{
+							Privileged: func(b bool) *bool { return &b }(true), // TODO https://stackoverflow.com/questions/28817992/how-to-set-bool-pointer-to-true-in-struct-literal
+						},
 					},
 				},
 				RestartPolicy: v1.RestartPolicyNever,