Skip to content

Commit

Permalink
test: Add E2E testing for fully blocking budgets with and without sch…
Browse files Browse the repository at this point in the history 
…edule (#5484)
  • Loading branch information
@jonathan-innis
jonathan-innis authored Jan 17, 2024
1 parent 529f243 commit a285991
Show file tree
Hide file tree
Showing 3 changed files with 192 additions and 31 deletions.
48 changes: 48 additions & 0 deletions test/suites/drift/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,54 @@ var _ = Describe("Drift", func() {
// the node should be gone
env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2])
})
It("should not allow drift if the budget is fully blocking", func() {
// We're going to define a budget that doesn't allow any drift to happen
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

By("drifting the nodes")
// Drift the nodeclaims
nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"}
env.ExpectUpdated(nodePool)

env.EventuallyExpectDrifted(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
It("should not allow drift if the budget is fully blocking during a scheduled time", func() {
// We're going to define a budget that doesn't allow any drift to happen
// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
// the current time and extends 15 minutes past the current time
// Times need to be in UTC since the karpenter containers were built in UTC time
windowStart := time.Now().Add(-time.Minute * 15).UTC()
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
Duration: &metav1.Duration{Duration: time.Minute * 30},
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

By("drifting the nodes")
// Drift the nodeclaims
nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"}
env.ExpectUpdated(nodePool)

env.EventuallyExpectDrifted(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
})
It("should disrupt nodes that have drifted due to AMIs", func() {
// choose an old static image
Expand Down
109 changes: 79 additions & 30 deletions test/suites/expiration/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ limitations under the License.
package expiration_test

import (
"fmt"
"testing"
"time"

"github.com/samber/lo"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -68,6 +70,28 @@ var _ = AfterEach(func() { env.Cleanup() })
var _ = AfterEach(func() { env.AfterEach() })

var _ = Describe("Expiration", func() {
var dep *appsv1.Deployment
var selector labels.Selector
var numPods int
BeforeEach(func() {
numPods = 1
// Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "my-app",
},
Annotations: map[string]string{
corev1beta1.DoNotDisruptAnnotationKey: "true",
},
},
TerminationGracePeriodSeconds: lo.ToPtr[int64](0),
},
})
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
})
Context("Budgets", func() {
It("should respect budgets for empty expiration", func() {
coretest.ReplaceRequirements(nodePool,
Expand All @@ -82,9 +106,9 @@ var _ = Describe("Expiration", func() {
}}
nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{}

var numPods int32 = 6
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
numPods = 6
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
Expand All @@ -100,12 +124,12 @@ var _ = Describe("Expiration", func() {
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3)
nodes := env.EventuallyExpectCreatedNodeCount("==", 3)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration

By("adding finalizers to the nodes to prevent termination")
Expand Down Expand Up @@ -159,9 +183,9 @@ var _ = Describe("Expiration", func() {
}}
// disable expiration so that we can enable it later when we want.
nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{}
var numPods int32 = 9
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
numPods = 9
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
Expand All @@ -177,12 +201,12 @@ var _ = Describe("Expiration", func() {
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3)
nodes := env.EventuallyExpectCreatedNodeCount("==", 3)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration

By("scaling down the deployment")
Expand Down Expand Up @@ -267,9 +291,9 @@ var _ = Describe("Expiration", func() {
Nodes: "50%",
}}
nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{}
var numPods int32 = 3
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
numPods = 3
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
Expand All @@ -285,12 +309,12 @@ var _ = Describe("Expiration", func() {
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3)
nodes := env.EventuallyExpectCreatedNodeCount("==", 3)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift

By("cordoning and adding finalizer to the nodes")
Expand Down Expand Up @@ -341,26 +365,51 @@ var _ = Describe("Expiration", func() {
// the node should be gone
env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2])
})
It("should not allow expiration if the budget is fully blocking", func() {
// We're going to define a budget that doesn't allow any expirations to happen
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

env.EventuallyExpectExpired(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
It("should not allow expiration if the budget is fully blocking during a scheduled time", func() {
// We're going to define a budget that doesn't allow any expirations to happen
// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
// the current time and extends 15 minutes past the current time
// Times need to be in UTC since the karpenter containers were built in UTC time
windowStart := time.Now().Add(-time.Minute * 15).UTC()
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
Duration: &metav1.Duration{Duration: time.Minute * 30},
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

env.EventuallyExpectExpired(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
})
It("should expire the node after the expiration is reached", func() {
var numPods int32 = 1
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
corev1beta1.DoNotDisruptAnnotationKey: "true",
},
Labels: map[string]string{"app": "large-app"},
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
node := env.EventuallyExpectCreatedNodeCount("==", 1)[0]
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration

env.EventuallyExpectExpired(nodeClaim)
Expand Down Expand Up @@ -392,7 +441,7 @@ var _ = Describe("Expiration", func() {

env.EventuallyExpectCreatedNodeClaimCount("==", 1)
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
})
It("should replace expired node with a single node and schedule all pods", func() {
var numPods int32 = 5
Expand Down
66 changes: 65 additions & 1 deletion test/suites/integration/emptiness_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,12 @@ limitations under the License.
package integration_test

import (
"fmt"
"time"

"github.com/samber/lo"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"knative.dev/pkg/ptr"

Expand All @@ -31,8 +34,69 @@ import (
)

var _ = Describe("Emptiness", func() {
It("should terminate an empty node", func() {
var dep *appsv1.Deployment
var selector labels.Selector
var numPods int
BeforeEach(func() {
nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty
nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))}

numPods = 1
dep = test.Deployment(test.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: test.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{"app": "large-app"},
},
},
})
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
})
Context("Budgets", func() {
It("should not allow emptiness if the budget is fully blocking", func() {
// We're going to define a budget that doesn't allow any emptiness disruption to happen
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
}}

env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

// Delete the deployment so there is nothing running on the node
env.ExpectDeleted(dep)

env.EventuallyExpectEmpty(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
It("should not allow emptiness if the budget is fully blocking during a scheduled time", func() {
// We're going to define a budget that doesn't allow any emptiness disruption to happen
// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
// the current time and extends 15 minutes past the current time
// Times need to be in UTC since the karpenter containers were built in UTC time
windowStart := time.Now().Add(-time.Minute * 15).UTC()
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
Duration: &metav1.Duration{Duration: time.Minute * 30},
}}

env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

// Delete the deployment so there is nothing running on the node
env.ExpectDeleted(dep)

env.EventuallyExpectEmpty(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
})
It("should terminate an empty node", func() {
nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Hour * 300)}

const numPods = 1
Expand Down

0 comments on commit a285991

Please sign in to comment.