diff --git a/test/suites/integration/storage_test.go b/test/suites/integration/storage_test.go index b289bc169a34..4eba427e12a8 100644 --- a/test/suites/integration/storage_test.go +++ b/test/suites/integration/storage_test.go @@ -15,105 +15,155 @@ limitations under the License. package integration_test import ( + "context" "fmt" - "github.com/aws/aws-sdk-go/aws" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/aws/aws-sdk-go/aws" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + "github.com/aws/karpenter/pkg/errors" + + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/test" ) -// This test requires the EBS CSI driver to be installed -var _ = Describe("Dynamic PVC", func() { - It("should run a pod with a dynamic persistent volume", func() { - // Ensure that the EBS driver is installed, or we can't run the test. - var ds appsv1.DaemonSet - if err := env.Client.Get(env.Context, client.ObjectKey{ - Namespace: "kube-system", - Name: "ebs-csi-node", - }, &ds); err != nil { - if errors.IsNotFound(err) { - Skip(fmt.Sprintf("skipping dynamic PVC test due to missing EBS driver %s", err)) - } else { - Fail(fmt.Sprintf("determining EBS driver status, %s", err)) - } - } - storageClassName := "ebs-sc-test" - bindMode := storagev1.VolumeBindingWaitForFirstConsumer - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: storageClassName, - }, - Provisioner: aws.String("ebs.csi.aws.com"), - VolumeBindingMode: &bindMode, +const IsDefaultStorageClassAnnotation = "storageclass.kubernetes.io/is-default-class" + +var _ = Describe("Persistent Volumes", func() { + Context("Static", func() { + It("should run a pod with a pre-bound persistent volume (empty storage class)", func() { + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + VolumeName: "test-volume", + StorageClassName: lo.ToPtr(""), + }) + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvc.Spec.VolumeName, + }, + }) + pod := test.Pod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + }) + + env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) }) - - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "ebs-claim", - }, - StorageClassName: aws.String(storageClassName), - Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}}, + It("should run a pod with a pre-bound persistent volume (non-existent storage class)", func() { + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + VolumeName: "test-volume", + StorageClassName: lo.ToPtr("non-existent-storage-class"), + }) + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvc.Spec.VolumeName, + }, + StorageClassName: "non-existent-storage-class", + }) + pod := test.Pod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + }) + env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) }) - - pod := test.Pod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name}, + It("should run a pod with a pre-bounnd persistent volume (empty storage class / no default)", func() { + restoreDefaults := ExpectDefaultStorageClassesRemoved(env.Context, env.Client) + defer restoreDefaults() + + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + VolumeName: "test-volume", + StorageClassName: lo.ToPtr(""), + }) + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: pvc.Spec.VolumeName, + }, + }) + pod := test.Pod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + }) + + env.ExpectCreated(nodeClass, nodePool, pv, pvc, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) }) - - env.ExpectCreated(nodeClass, nodePool, sc, pvc, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - env.ExpectDeleted(pod) }) -}) -var _ = Describe("Static PVC", func() { - It("should run a pod with a static persistent volume", func() { - storageClassName := "nfs-test" - bindMode := storagev1.VolumeBindingWaitForFirstConsumer - sc := test.StorageClass(test.StorageClassOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: storageClassName, - }, - VolumeBindingMode: &bindMode, - }) - - pv := test.PersistentVolume(test.PersistentVolumeOptions{ - ObjectMeta: metav1.ObjectMeta{Name: "nfs-test-volume"}, - StorageClassName: "nfs-test", + Context("Dynamic", func() { + var storageClass *storagev1.StorageClass + BeforeEach(func() { + // Ensure that the EBS driver is installed, or we can't run the test. + var ds appsv1.DaemonSet + if err := env.Client.Get(env.Context, client.ObjectKey{ + Namespace: "kube-system", + Name: "ebs-csi-node", + }, &ds); err != nil { + if errors.IsNotFound(err) { + Skip(fmt.Sprintf("skipping dynamic PVC test due to missing EBS driver %s", err)) + } else { + Fail(fmt.Sprintf("determining EBS driver status, %s", err)) + } + } + storageClass = test.StorageClass(test.StorageClassOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-storage-class", + }, + Provisioner: aws.String("ebs.csi.aws.com"), + VolumeBindingMode: lo.ToPtr(storagev1.VolumeBindingWaitForFirstConsumer), + }) }) - - // the server here doesn't need to actually exist for the pod to start running - pv.Spec.NFS = &v1.NFSVolumeSource{ - Server: "fake.server", - Path: "/some/path", - } - pv.Spec.CSI = nil - - pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ - ObjectMeta: metav1.ObjectMeta{ - Name: "nfs-claim", - }, - StorageClassName: aws.String(storageClassName), - VolumeName: pv.Name, - Resources: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("5Gi")}}, + It("should run a pod with a dynamic persistent volume (non-existent storage class)", func() { + pvc := test.PersistentVolumeClaim(test.PersistentVolumeClaimOptions{ + StorageClassName: &storageClass.Name, + }) + pod := test.Pod(test.PodOptions{ + PersistentVolumeClaims: []string{pvc.Name}, + }) + + env.ExpectCreated(nodeClass, nodePool, storageClass, pvc, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) }) - - pod := test.Pod(test.PodOptions{ - PersistentVolumeClaims: []string{pvc.Name}, + It("should run a pod with a generic ephemeral volume", func() { + pod := test.Pod(test.PodOptions{ + EphemeralVolumeTemplates: []test.EphemeralVolumeTemplateOptions{{ + StorageClassName: &storageClass.Name, + }}, + }) + + env.ExpectCreated(nodeClass, nodePool, storageClass, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) }) - - env.ExpectCreated(nodeClass, nodePool, sc, pv, pvc, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - env.ExpectDeleted(pod) }) }) + +func ExpectDefaultStorageClassesRemoved(ctx context.Context, kubeClient client.Client) func() { + classes := &storagev1.StorageClassList{} + Expect(kubeClient.List(ctx, classes)).To(Succeed()) + defaultClasses := lo.Map(lo.Filter(classes.Items, func(sc storagev1.StorageClass, _ int) bool { + return sc.Annotations[IsDefaultStorageClassAnnotation] == "true" + }), func(sc storagev1.StorageClass, _ int) *storagev1.StorageClass { + return lo.ToPtr(sc) + }) + for _, sc := range defaultClasses { + persisted := sc.DeepCopy() + delete(sc.Annotations, IsDefaultStorageClassAnnotation) + Expect(kubeClient.Patch(ctx, sc, client.MergeFrom(persisted))).To(Succeed()) + } + + return func() { + for _, sc := range defaultClasses { + persisted := sc.DeepCopy() + sc.Annotations[IsDefaultStorageClassAnnotation] = "true" + Expect(kubeClient.Patch(ctx, sc, client.MergeFrom(persisted))).To(Succeed()) + } + } +}