From 46e72698feaffadc3174fa77acccab27fba8e894 Mon Sep 17 00:00:00 2001
From: Jason Deal <jmdeal@amazon.com>
Date: Mon, 4 Dec 2023 17:30:45 -0800
Subject: [PATCH] add integration tests

---
 test/suites/integration/storage_test.go | 212 +++++++++++++++---------
 1 file changed, 131 insertions(+), 81 deletions(-)

diff --git a/test/suites/integration/storage_test.go b/test/suites/integration/storage_test.go
index b289bc169a34..4eba427e12a8 100644
--- a/test/suites/integration/storage_test.go
+++ b/test/suites/integration/storage_test.go
@@ -15,105 +15,155 @@ limitations under the License.
 package integration_test
 
 import (
+	"context"
 	"fmt"
 
-	"github.com/aws/aws-sdk-go/aws"
 	appsv1 "k8s.io/api/apps/v1"
-	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"github.com/aws/aws-sdk-go/aws"
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"github.com/samber/lo"
 
+	"github.com/aws/karpenter/pkg/errors"
+
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/karpenter/pkg/test"
 )
 
-// This test requires the EBS CSI driver to be installed
-var _ = Describe("Dynamic PVC", func() {
-	It("should run a pod with a dynamic persistent volume", func() {
-		// Ensure that the EBS driver is installed, or we can't run the test.
-		var ds appsv1.DaemonSet
-		if err := env.Client.Get(env.Context, client.ObjectKey{
-			Namespace: "kube-system",
-			Name:      "ebs-csi-node",
-		}, &ds); err != nil {
-			if errors.IsNotFound(err) {
-				Skip(fmt.Sprintf("skipping dynamic PVC test due to missing EBS driver %s", err))
-			} else {
-				Fail(fmt.Sprintf("determining EBS driver status, %s", err))
-			}
-		}
-		storageClassName := "ebs-sc-test"
-		bindMode := storagev1.VolumeBindingWaitForFirstConsumer
-		sc := test.StorageClass(test.StorageClassOptions{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: storageClassName,
-			},
-			Provisioner:       aws.String("ebs.csi.aws.com"),
-			VolumeBindingMode: &bindMode,
+const IsDefaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class"
+
+var _ = Describe("Persistent Volumes", func() {
+	Context("Static", func() {
+		It("should run a pod with a pre-bound persistent volume (empty storage class)", func() {
+			pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
+				VolumeName:       "test-volume",
+				StorageClassName: lo.ToPtr(""),
+			})
+			pv := test.PersistentVolume(test.PersistentVolumeOptions{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: pvc.Spec.VolumeName,
+				},
+			})
+			pod := test.Pod(test.PodOptions{
+				PersistentVolumeClaims: []string{pvc.Name},
+			})
+
+			env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod)
+			env.EventuallyExpectHealthy(pod)
+			env.ExpectCreatedNodeCount("==", 1)
 		})
-
-		pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "ebs-claim",
-			},
-			StorageClassName: aws.String(storageClassName),
-			Resources:        v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}},
+		It("should run a pod with a pre-bound persistent volume (non-existent storage class)", func() {
+			pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
+				VolumeName:       "test-volume",
+				StorageClassName: lo.ToPtr("non-existent-storage-class"),
+			})
+			pv := test.PersistentVolume(test.PersistentVolumeOptions{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: pvc.Spec.VolumeName,
+				},
+				StorageClassName: "non-existent-storage-class",
+			})
+			pod := test.Pod(test.PodOptions{
+				PersistentVolumeClaims: []string{pvc.Name},
+			})
+			env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod)
+			env.EventuallyExpectHealthy(pod)
+			env.ExpectCreatedNodeCount("==", 1)
 		})
-
-		pod := test.Pod(test.PodOptions{
-			PersistentVolumeClaims: []string{pvc.Name},
+		It("should run a pod with a pre-bounnd persistent volume (empty storage class / no default)", func() {
+			restoreDefaults := ExpectDefaultStorageClassesRemoved(env.Context, env.Client)
+			defer restoreDefaults()
+
+			pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
+				VolumeName:       "test-volume",
+				StorageClassName: lo.ToPtr(""),
+			})
+			pv := test.PersistentVolume(test.PersistentVolumeOptions{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: pvc.Spec.VolumeName,
+				},
+			})
+			pod := test.Pod(test.PodOptions{
+				PersistentVolumeClaims: []string{pvc.Name},
+			})
+
+			env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod)
+			env.EventuallyExpectHealthy(pod)
+			env.ExpectCreatedNodeCount("==", 1)
 		})
-
-		env.ExpectCreated(nodeClass, nodePool, sc, pvc, pod)
-		env.EventuallyExpectHealthy(pod)
-		env.ExpectCreatedNodeCount("==", 1)
-		env.ExpectDeleted(pod)
 	})
-})
 
-var _ = Describe("Static PVC", func() {
-	It("should run a pod with a static persistent volume", func() {
-		storageClassName := "nfs-test"
-		bindMode := storagev1.VolumeBindingWaitForFirstConsumer
-		sc := test.StorageClass(test.StorageClassOptions{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: storageClassName,
-			},
-			VolumeBindingMode: &bindMode,
-		})
-
-		pv := test.PersistentVolume(test.PersistentVolumeOptions{
-			ObjectMeta:       metav1.ObjectMeta{Name: "nfs-test-volume"},
-			StorageClassName: "nfs-test",
+	Context("Dynamic", func() {
+		var storageClass *storagev1.StorageClass
+		BeforeEach(func() {
+			// Ensure that the EBS driver is installed, or we can't run the test.
+			var ds appsv1.DaemonSet
+			if err := env.Client.Get(env.Context, client.ObjectKey{
+				Namespace: "kube-system",
+				Name:      "ebs-csi-node",
+			}, &ds); err != nil {
+				if errors.IsNotFound(err) {
+					Skip(fmt.Sprintf("skipping dynamic PVC test due to missing EBS driver %s", err))
+				} else {
+					Fail(fmt.Sprintf("determining EBS driver status, %s", err))
+				}
+			}
+			storageClass = test.StorageClass(test.StorageClassOptions{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test-storage-class",
+				},
+				Provisioner:       aws.String("ebs.csi.aws.com"),
+				VolumeBindingMode: lo.ToPtr(storagev1.VolumeBindingWaitForFirstConsumer),
+			})
 		})
-
-		// the server here doesn't need to actually exist for the pod to start running
-		pv.Spec.NFS = &v1.NFSVolumeSource{
-			Server: "fake.server",
-			Path:   "/some/path",
-		}
-		pv.Spec.CSI = nil
-
-		pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
-			ObjectMeta: metav1.ObjectMeta{
-				Name: "nfs-claim",
-			},
-			StorageClassName: aws.String(storageClassName),
-			VolumeName:       pv.Name,
-			Resources:        v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}},
+		It("should run a pod with a dynamic persistent volume (non-existent storage class)", func() {
+			pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{
+				StorageClassName: &storageClass.Name,
+			})
+			pod := test.Pod(test.PodOptions{
+				PersistentVolumeClaims: []string{pvc.Name},
+			})
+
+			env.ExpectCreated(nodeClass, nodePool, storageClass, pvc, pod)
+			env.EventuallyExpectHealthy(pod)
+			env.ExpectCreatedNodeCount("==", 1)
 		})
-
-		pod := test.Pod(test.PodOptions{
-			PersistentVolumeClaims: []string{pvc.Name},
+		It("should run a pod with a generic ephemeral volume", func() {
+			pod := test.Pod(test.PodOptions{
+				EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{{
+					StorageClassName: &storageClass.Name,
+				}},
+			})
+
+			env.ExpectCreated(nodeClass, nodePool, storageClass, pod)
+			env.EventuallyExpectHealthy(pod)
+			env.ExpectCreatedNodeCount("==", 1)
 		})
-
-		env.ExpectCreated(nodeClass, nodePool, sc, pv, pvc, pod)
-		env.EventuallyExpectHealthy(pod)
-		env.ExpectCreatedNodeCount("==", 1)
-		env.ExpectDeleted(pod)
 	})
 })
+
+func ExpectDefaultStorageClassesRemoved(ctx context.Context, kubeClient client.Client) func() {
+	classes := &storagev1.StorageClassList{}
+	Expect(kubeClient.List(ctx, classes)).To(Succeed())
+	defaultClasses := lo.Map(lo.Filter(classes.Items, func(sc storagev1.StorageClass, _ int) bool {
+		return sc.Annotations[IsDefaultStorageClassAnnotation] == "true"
+	}), func(sc storagev1.StorageClass, _ int) *storagev1.StorageClass {
+		return lo.ToPtr(sc)
+	})
+	for _, sc := range defaultClasses {
+		persisted := sc.DeepCopy()
+		delete(sc.Annotations, IsDefaultStorageClassAnnotation)
+		Expect(kubeClient.Patch(ctx, sc, client.MergeFrom(persisted))).To(Succeed())
+	}
+
+	return func() {
+		for _, sc := range defaultClasses {
+			persisted := sc.DeepCopy()
+			sc.Annotations[IsDefaultStorageClassAnnotation] = "true"
+			Expect(kubeClient.Patch(ctx, sc, client.MergeFrom(persisted))).To(Succeed())
+		}
+	}
+}