diff --git a/pkg/controllers/termination/suite_test.go b/pkg/controllers/termination/suite_test.go index 915a5df6afa1..d0352015fcc3 100644 --- a/pkg/controllers/termination/suite_test.go +++ b/pkg/controllers/termination/suite_test.go @@ -34,6 +34,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" . "knative.dev/pkg/logging/testing" @@ -196,6 +197,48 @@ var _ = Describe("Termination", func() { ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(node)) ExpectNotFound(ctx, env.Client, node) }) + It("should not evict static pods", func() { + podEvict := test.Pod(test.PodOptions{NodeName: node.Name}) + ExpectCreated(ctx, env.Client, node, podEvict) + + podNoEvict := test.Pod(test.PodOptions{ + NodeName: node.Name, + OwnerReferences: []metav1.OwnerReference{{ + APIVersion: "v1", + Kind: "Node", + Name: node.Name, + UID: node.UID, + }}, + }) + ExpectCreated(ctx, env.Client, podNoEvict) + + Expect(env.Client.Delete(ctx, node)).To(Succeed()) + node = ExpectNodeExists(ctx, env.Client, node.Name) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(node)) + + // Expect mirror pod to not be queued for enviction + ExpectNotEnqueuedForEviction(evictionQueue, podNoEvict) + + // Expect podEvict to be enqueued for eviction then be successful + ExpectEnqueuedForEviction(evictionQueue, podEvict) + ExpectEvicted(env.Client, podEvict) + + // Expect node to exist and be draining + ExpectNodeDraining(env.Client, node.Name) + + // Reconcile node to evict pod + node = ExpectNodeExists(ctx, env.Client, node.Name) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(node)) + + // Delete pod to simulate successful eviction + ExpectDeleted(ctx, env.Client, podEvict) + + // Reconcile to delete node + node = ExpectNodeExists(ctx, env.Client, node.Name) + ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(node)) + ExpectNotFound(ctx, env.Client, node) + + }) It("should not delete nodes until all pods are deleted", func() { pods := []*v1.Pod{test.Pod(test.PodOptions{NodeName: node.Name}), test.Pod(test.PodOptions{NodeName: node.Name})} ExpectCreated(ctx, env.Client, node, pods[0], pods[1]) diff --git a/pkg/controllers/termination/terminate.go b/pkg/controllers/termination/terminate.go index 75ef6a8ca564..b4f4f02f56ac 100644 --- a/pkg/controllers/termination/terminate.go +++ b/pkg/controllers/termination/terminate.go @@ -28,6 +28,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/utils/functional" "github.com/aws/karpenter/pkg/utils/injectabletime" + "github.com/aws/karpenter/pkg/utils/pod" "github.com/aws/karpenter/pkg/utils/ptr" ) @@ -110,16 +111,20 @@ func (t *Terminator) getPods(ctx context.Context, node *v1.Node) ([]*v1.Pod, err func (t *Terminator) getEvictablePods(pods []*v1.Pod) []*v1.Pod { evictable := []*v1.Pod{} - for _, pod := range pods { + for _, p := range pods { // Ignore if unschedulable is tolerated, since they will reschedule - if (v1alpha5.Taints{{Key: v1.TaintNodeUnschedulable, Effect: v1.TaintEffectNoSchedule}}).Tolerates(pod) == nil { + if (v1alpha5.Taints{{Key: v1.TaintNodeUnschedulable, Effect: v1.TaintEffectNoSchedule}}).Tolerates(p) == nil { continue } // Ignore if kubelet is partitioned and pods are beyond graceful termination window - if IsStuckTerminating(pod) { + if IsStuckTerminating(p) { + continue + } + // Ignore static mirror pods + if pod.IsOwnedByNode(p) { continue } - evictable = append(evictable, pod) + evictable = append(evictable, p) } return evictable }