From a285991f9a7617068a01d079f53e5d6da7eee61f Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Wed, 17 Jan 2024 10:31:54 -0800 Subject: [PATCH] test: Add E2E testing for fully blocking budgets with and without schedule (#5484) --- test/suites/drift/suite_test.go | 48 ++++++++++ test/suites/expiration/suite_test.go | 109 ++++++++++++++++------ test/suites/integration/emptiness_test.go | 66 ++++++++++++- 3 files changed, 192 insertions(+), 31 deletions(-) diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index d3d600363c29..6a3d65037fc8 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -361,6 +361,54 @@ var _ = Describe("Drift", func() { // the node should be gone env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) }) + It("should not allow drift if the budget is fully blocking", func() { + // We're going to define a budget that doesn't allow any drift to happen + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + By("drifting the nodes") + // Drift the nodeclaims + nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + env.ExpectUpdated(nodePool) + + env.EventuallyExpectDrifted(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + It("should not allow drift if the budget is fully blocking during a scheduled time", func() { + // We're going to define a budget that doesn't allow any drift to happen + // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before + // the current time and extends 15 minutes past the current time + // Times need to be in UTC since the karpenter containers were built in UTC time + windowStart := time.Now().Add(-time.Minute * 15).UTC() + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), + Duration: &metav1.Duration{Duration: time.Minute * 30}, + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + By("drifting the nodes") + // Drift the nodeclaims + nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + env.ExpectUpdated(nodePool) + + env.EventuallyExpectDrifted(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) }) It("should disrupt nodes that have drifted due to AMIs", func() { // choose an old static image diff --git a/test/suites/expiration/suite_test.go b/test/suites/expiration/suite_test.go index 27a4ea28a4c2..6ceb1c365f8d 100644 --- a/test/suites/expiration/suite_test.go +++ b/test/suites/expiration/suite_test.go @@ -15,10 +15,12 @@ limitations under the License. package expiration_test import ( + "fmt" "testing" "time" "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -68,6 +70,28 @@ var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Expiration", func() { + var dep *appsv1.Deployment + var selector labels.Selector + var numPods int + BeforeEach(func() { + numPods = 1 + // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "my-app", + }, + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + }, + TerminationGracePeriodSeconds: lo.ToPtr[int64](0), + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + }) Context("Budgets", func() { It("should respect budgets for empty expiration", func() { coretest.ReplaceRequirements(nodePool, @@ -82,9 +106,9 @@ var _ = Describe("Expiration", func() { }} nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - var numPods int32 = 6 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, + numPods = 6 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -100,12 +124,12 @@ var _ = Describe("Expiration", func() { }, }, }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) nodes := env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration By("adding finalizers to the nodes to prevent termination") @@ -159,9 +183,9 @@ var _ = Describe("Expiration", func() { }} // disable expiration so that we can enable it later when we want. nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - var numPods int32 = 9 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, + numPods = 9 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -177,12 +201,12 @@ var _ = Describe("Expiration", func() { }, }, }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) nodes := env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration By("scaling down the deployment") @@ -267,9 +291,9 @@ var _ = Describe("Expiration", func() { Nodes: "50%", }} nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - var numPods int32 = 3 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, + numPods = 3 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -285,12 +309,12 @@ var _ = Describe("Expiration", func() { }, }, }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) nodes := env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift By("cordoning and adding finalizer to the nodes") @@ -341,26 +365,51 @@ var _ = Describe("Expiration", func() { // the node should be gone env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) }) + It("should not allow expiration if the budget is fully blocking", func() { + // We're going to define a budget that doesn't allow any expirations to happen + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + env.EventuallyExpectExpired(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + It("should not allow expiration if the budget is fully blocking during a scheduled time", func() { + // We're going to define a budget that doesn't allow any expirations to happen + // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before + // the current time and extends 15 minutes past the current time + // Times need to be in UTC since the karpenter containers were built in UTC time + windowStart := time.Now().Add(-time.Minute * 15).UTC() + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), + Duration: &metav1.Duration{Duration: time.Minute * 30}, + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + env.EventuallyExpectExpired(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) }) It("should expire the node after the expiration is reached", func() { - var numPods int32 = 1 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, - Labels: map[string]string{"app": "large-app"}, - }, - }, - }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration env.EventuallyExpectExpired(nodeClaim) @@ -392,7 +441,7 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectCreatedNodeClaimCount("==", 1) env.EventuallyExpectCreatedNodeCount("==", 1) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should replace expired node with a single node and schedule all pods", func() { var numPods int32 = 5 diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go index d1ba42d3b18c..d583e4a07b11 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/integration/emptiness_test.go @@ -15,9 +15,12 @@ limitations under the License. package integration_test import ( + "fmt" "time" "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "knative.dev/pkg/ptr" @@ -31,8 +34,69 @@ import ( ) var _ = Describe("Emptiness", func() { - It("should terminate an empty node", func() { + var dep *appsv1.Deployment + var selector labels.Selector + var numPods int + BeforeEach(func() { nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))} + + numPods = 1 + dep = test.Deployment(test.DeploymentOptions{ + Replicas: int32(numPods), + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "large-app"}, + }, + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + }) + Context("Budgets", func() { + It("should not allow emptiness if the budget is fully blocking", func() { + // We're going to define a budget that doesn't allow any emptiness disruption to happen + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + }} + + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + // Delete the deployment so there is nothing running on the node + env.ExpectDeleted(dep) + + env.EventuallyExpectEmpty(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + It("should not allow emptiness if the budget is fully blocking during a scheduled time", func() { + // We're going to define a budget that doesn't allow any emptiness disruption to happen + // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before + // the current time and extends 15 minutes past the current time + // Times need to be in UTC since the karpenter containers were built in UTC time + windowStart := time.Now().Add(-time.Minute * 15).UTC() + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), + Duration: &metav1.Duration{Duration: time.Minute * 30}, + }} + + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + // Delete the deployment so there is nothing running on the node + env.ExpectDeleted(dep) + + env.EventuallyExpectEmpty(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + }) + It("should terminate an empty node", func() { nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Hour * 300)} const numPods = 1