From debbc3a4de08f7283c491e0223d5db14412edc22 Mon Sep 17 00:00:00 2001 From: Claudio Busse Date: Tue, 4 Jun 2024 11:48:07 +0200 Subject: [PATCH] fix: delayed MHC replacement of unreachable nodes Co-authored-by: Michael Shen --- internal/controllers/machine/machine_controller.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/controllers/machine/machine_controller.go b/internal/controllers/machine/machine_controller.go index 4ac3566c6405..318c712a92d5 100644 --- a/internal/controllers/machine/machine_controller.go +++ b/internal/controllers/machine/machine_controller.go @@ -670,8 +670,18 @@ func (r *Reconciler) drainNode(ctx context.Context, cluster *clusterv1.Cluster, } if noderefutil.IsNodeUnreachable(node) { - // When the node is unreachable and some pods are not evicted for as long as this timeout, we ignore them. - drainer.SkipWaitForDeleteTimeoutSeconds = 60 * 5 // 5 minutes + // Kubelet is unreachable, pods will never disappear. + + // SkipWaitForDeleteTimeoutSeconds ensures the drain completes + // even if pod objects are not deleted. + drainer.SkipWaitForDeleteTimeoutSeconds = 1 + + // kube-apiserver sets the `deletionTimestamp` to a future date computed using the grace period. + // We are effectively waiting for GracePeriodSeconds + SkipWaitForDeleteTimeoutSeconds. + // Override the grace period of pods to reduce the time needed to skip them. + drainer.GracePeriodSeconds = 1 + + log.V(5).Info("Node is unreachable, draining will ignore gracePeriod. PDBs are still honored.") } if err := kubedrain.RunCordonOrUncordon(drainer, node, true); err != nil {