Handle pod eviction errors correctly (eksctl-io#5116)

* Handle pod eviction errors correctly

Currently any eviction error causes the draining of a node to stop and a new
node to start draining.  Eviction errors are common, expected occurences
especially when PDBs are used in the cluster.
By having any error abort the draining of a node we slow down the entire node
draining process as many of the pods further in the list could happily be
drained.

This change separates recoverable and irrecoverable eviction errors and retries
only the recoverable.  Unrecoverable errors fail the entire command.

An important aspect of this is that the `evictPods` function becomes blocking
until a node is drained or the process times out.  This is required as the
current implementation begins draining another node on the first eviction
error.  We would rather keep trying and eventually time out than make a bad
situation worse by draining a new node.

* Update pkg/drain/nodegroup.go

Co-authored-by: Gergely Brautigam <[email protected]>

* Update pkg/drain/nodegroup.go

Co-authored-by: Gergely Brautigam <[email protected]>

* Update pkg/drain/nodegroup.go

Co-authored-by: Gergely Brautigam <[email protected]>

* Update pkg/drain/nodegroup.go

Co-authored-by: Gergely Brautigam <[email protected]>

* Update pkg/drain/nodegroup.go

Co-authored-by: Gergely Brautigam <[email protected]>

* Update pkg/drain/nodegroup.go

Co-authored-by: Gergely Brautigam <[email protected]>

* Use apimachinery error functions to check for recoverable errors

* Add --pod-eviction-wait-period flag to cluster delete command

Co-authored-by: William Hutcheson <[email protected]>
Co-authored-by: Gergely Brautigam <[email protected]>

pkg/actions/cluster/cluster.go

@@ -17,7 +17,7 @@ import (
 
 type Cluster interface {
 	Upgrade(ctx context.Context, dryRun bool) error
-	Delete(ctx context.Context, waitInterval time.Duration, wait, force, disableNodegroupEviction bool, parallel int) error
+	Delete(ctx context.Context, waitInterval, podEvictionWaitPeriod time.Duration, wait, force, disableNodegroupEviction bool, parallel int) error
 }
 
 func New(ctx context.Context, cfg *api.ClusterConfig, ctl *eks.ClusterProvider) (Cluster, error) {

pkg/actions/cluster/delete.go

@@ -166,7 +166,7 @@ func checkForUndeletedStacks(ctx context.Context, stackManager manager.StackMana
 }
 
 func drainAllNodeGroups(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, clientSet kubernetes.Interface, allStacks []manager.NodeGroupStack,
-	disableEviction bool, parallel int, nodeGroupDrainer NodeGroupDrainer, vpcCniDeleter vpcCniDeleter) error {
+	disableEviction bool, parallel int, nodeGroupDrainer NodeGroupDrainer, vpcCniDeleter vpcCniDeleter, podEvictionWaitPeriod time.Duration) error {
 	if len(allStacks) == 0 {
 		return nil
 	}
@@ -181,10 +181,11 @@ func drainAllNodeGroups(cfg *api.ClusterConfig, ctl *eks.ClusterProvider, client
 	logger.Info("will drain %d unmanaged nodegroup(s) in cluster %q", len(cfg.NodeGroups), cfg.Metadata.Name)
 
 	drainInput := &nodegroup.DrainInput{
-		NodeGroups:      cmdutils.ToKubeNodeGroups(cfg),
-		MaxGracePeriod:  ctl.Provider.WaitTimeout(),
-		DisableEviction: disableEviction,
-		Parallel:        parallel,
+		NodeGroups:            cmdutils.ToKubeNodeGroups(cfg),
+		MaxGracePeriod:        ctl.Provider.WaitTimeout(),
+		DisableEviction:       disableEviction,
+		PodEvictionWaitPeriod: podEvictionWaitPeriod,
+		Parallel:              parallel,
 	}
 	if err := nodeGroupDrainer.Drain(drainInput); err != nil {
 		return err

pkg/actions/cluster/delete_test.go

@@ -1,6 +1,8 @@
 package cluster_test
 
 import (
+	"time"
+
 	"github.com/weaveworks/eksctl/pkg/actions/nodegroup"
 	"github.com/weaveworks/eksctl/pkg/ctl/cmdutils"
 
@@ -68,7 +70,7 @@ var _ = Describe("DrainAllNodeGroups", func() {
 					vpcCniDeleterCalled++
 				}
 
-				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, false, 1, mockedDrainer, vpcCniDeleter)
+				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, false, 1, mockedDrainer, vpcCniDeleter, time.Second*0)
 				Expect(err).NotTo(HaveOccurred())
 				mockedDrainer.AssertNumberOfCalls(GinkgoT(), "Drain", 1)
 				Expect(vpcCniDeleterCalled).To(Equal(1))
@@ -97,7 +99,7 @@ var _ = Describe("DrainAllNodeGroups", func() {
 					vpcCniDeleterCalled++
 				}
 
-				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, true, 1, mockedDrainer, vpcCniDeleter)
+				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, true, 1, mockedDrainer, vpcCniDeleter, time.Second*0)
 				Expect(err).NotTo(HaveOccurred())
 				mockedDrainer.AssertNumberOfCalls(GinkgoT(), "Drain", 1)
 				Expect(vpcCniDeleterCalled).To(Equal(1))
@@ -125,7 +127,7 @@ var _ = Describe("DrainAllNodeGroups", func() {
 					vpcCniDeleterCalled++
 				}
 
-				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, false, 1, mockedDrainer, vpcCniDeleter)
+				err := cluster.DrainAllNodeGroups(cfg, ctl, fakeClientSet, nodeGroupStacks, false, 1, mockedDrainer, vpcCniDeleter, time.Second*0)
 				Expect(err).NotTo(HaveOccurred())
 				mockedDrainer.AssertNotCalled(GinkgoT(), "Drain")
 				Expect(vpcCniDeleterCalled).To(Equal(0))

pkg/actions/cluster/owned.go

@@ -65,7 +65,7 @@ func (c *OwnedCluster) Upgrade(ctx context.Context, dryRun bool) error {
 	return nil
 }
 
-func (c *OwnedCluster) Delete(ctx context.Context, _ time.Duration, wait, force, disableNodegroupEviction bool, parallel int) error {
+func (c *OwnedCluster) Delete(ctx context.Context, _, podEvictionWaitPeriod time.Duration, wait, force, disableNodegroupEviction bool, parallel int) error {
 	var (
 		clientSet kubernetes.Interface
 		oidc      *iamoidc.OpenIDConnectManager
@@ -107,7 +107,7 @@ func (c *OwnedCluster) Delete(ctx context.Context, _ time.Duration, wait, force,
 		}
 
 		nodeGroupManager := c.newNodeGroupManager(c.cfg, c.ctl, clientSet)
-		if err := drainAllNodeGroups(c.cfg, c.ctl, clientSet, allStacks, disableNodegroupEviction, parallel, nodeGroupManager, attemptVpcCniDeletion); err != nil {
+		if err := drainAllNodeGroups(c.cfg, c.ctl, clientSet, allStacks, disableNodegroupEviction, parallel, nodeGroupManager, attemptVpcCniDeletion, podEvictionWaitPeriod); err != nil {
 			if !force {
 				return err
 			}

pkg/actions/cluster/owned_test.go

@@ -114,7 +114,7 @@ var _ = Describe("Delete", func() {
 				return fakeClientSet, nil
 			})
 
-			err := c.Delete(context.Background(), time.Microsecond, false, false, false, 1)
+			err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, false, false, 1)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(fakeStackManager.DeleteTasksForDeprecatedStacksCallCount()).To(Equal(1))
 			Expect(ranDeleteDeprecatedTasks).To(BeTrue())
@@ -189,7 +189,7 @@ var _ = Describe("Delete", func() {
 					return mockedDrainer
 				})
 
-				err := c.Delete(context.Background(), time.Microsecond, false, true, false, 1)
+				err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, true, false, 1)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(fakeStackManager.DeleteTasksForDeprecatedStacksCallCount()).To(Equal(1))
 				Expect(ranDeleteDeprecatedTasks).To(BeFalse())
@@ -262,7 +262,7 @@ var _ = Describe("Delete", func() {
 					return mockedDrainer
 				})
 
-				err := c.Delete(context.Background(), time.Microsecond, false, false, false, 1)
+				err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, false, false, 1)
 				Expect(err).To(MatchError(errorMessage))
 				Expect(fakeStackManager.DeleteTasksForDeprecatedStacksCallCount()).To(Equal(0))
 				Expect(ranDeleteDeprecatedTasks).To(BeFalse())
@@ -303,7 +303,7 @@ var _ = Describe("Delete", func() {
 
 			c := cluster.NewOwnedCluster(cfg, ctl, nil, fakeStackManager)
 
-			err := c.Delete(context.Background(), time.Microsecond, false, false, false, 1)
+			err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, false, false, 1)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(fakeStackManager.DeleteTasksForDeprecatedStacksCallCount()).To(Equal(1))
 			Expect(ranDeleteDeprecatedTasks).To(BeTrue())

pkg/actions/cluster/unowned.go

@@ -55,7 +55,7 @@ func (c *UnownedCluster) Upgrade(_ context.Context, dryRun bool) error {
 	return nil
 }
 
-func (c *UnownedCluster) Delete(ctx context.Context, waitInterval time.Duration, wait, force, disableNodegroupEviction bool, parallel int) error {
+func (c *UnownedCluster) Delete(ctx context.Context, waitInterval, podEvictionWaitPeriod time.Duration, wait, force, disableNodegroupEviction bool, parallel int) error {
 	clusterName := c.cfg.Metadata.Name
 
 	if err := c.checkClusterExists(clusterName); err != nil {
@@ -80,7 +80,7 @@ func (c *UnownedCluster) Delete(ctx context.Context, waitInterval time.Duration,
 		}
 
 		nodeGroupManager := c.newNodeGroupManager(c.cfg, c.ctl, clientSet)
-		if err := drainAllNodeGroups(c.cfg, c.ctl, clientSet, allStacks, disableNodegroupEviction, parallel, nodeGroupManager, attemptVpcCniDeletion); err != nil {
+		if err := drainAllNodeGroups(c.cfg, c.ctl, clientSet, allStacks, disableNodegroupEviction, parallel, nodeGroupManager, attemptVpcCniDeletion, podEvictionWaitPeriod); err != nil {
 			if !force {
 				return err
 			}

pkg/actions/cluster/unowned_test.go

@@ -152,7 +152,7 @@ var _ = Describe("Delete", func() {
 				return fakeClientSet, nil
 			})
 
-			err := c.Delete(context.Background(), time.Microsecond, false, false, false, 1)
+			err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, false, false, 1)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(deleteCallCount).To(Equal(1))
 			Expect(unownedDeleteCallCount).To(Equal(1))
@@ -255,7 +255,7 @@ var _ = Describe("Delete", func() {
 					return mockedDrainer
 				})
 
-				err := c.Delete(context.Background(), time.Microsecond, false, true, false, 1)
+				err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, true, false, 1)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(deleteCallCount).To(Equal(0))
 				Expect(unownedDeleteCallCount).To(Equal(0))
@@ -358,7 +358,7 @@ var _ = Describe("Delete", func() {
 					return mockedDrainer
 				})
 
-				err := c.Delete(context.Background(), time.Microsecond, false, false, false, 1)
+				err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, false, false, 1)
 				Expect(err).To(MatchError(errorMessage))
 				Expect(deleteCallCount).To(Equal(0))
 				Expect(unownedDeleteCallCount).To(Equal(0))
@@ -422,7 +422,7 @@ var _ = Describe("Delete", func() {
 			p.MockEKS().On("DeleteCluster", mock.Anything).Return(&awseks.DeleteClusterOutput{}, nil)
 
 			c := cluster.NewUnownedCluster(cfg, ctl, fakeStackManager)
-			err := c.Delete(context.Background(), time.Microsecond, false, false, false, 1)
+			err := c.Delete(context.Background(), time.Microsecond, time.Second*0, false, false, false, 1)
 			Expect(err).NotTo(HaveOccurred())
 			Expect(fakeStackManager.DeleteTasksForDeprecatedStacksCallCount()).To(Equal(1))
 			Expect(deleteCallCount).To(Equal(1))

pkg/actions/nodegroup/drain.go

@@ -9,19 +9,20 @@ import (
 )
 
 type DrainInput struct {
-	NodeGroups          []eks.KubeNodeGroup
-	Plan                bool
-	MaxGracePeriod      time.Duration
-	NodeDrainWaitPeriod time.Duration
-	Undo                bool
-	DisableEviction     bool
-	Parallel            int
+	NodeGroups            []eks.KubeNodeGroup
+	Plan                  bool
+	MaxGracePeriod        time.Duration
+	NodeDrainWaitPeriod   time.Duration
+	PodEvictionWaitPeriod time.Duration
+	Undo                  bool
+	DisableEviction       bool
+	Parallel              int
 }
 
 func (m *Manager) Drain(input *DrainInput) error {
 	if !input.Plan {
 		for _, n := range input.NodeGroups {
-			nodeGroupDrainer := drain.NewNodeGroupDrainer(m.clientSet, n, m.ctl.Provider.WaitTimeout(), input.MaxGracePeriod, input.NodeDrainWaitPeriod, input.Undo, input.DisableEviction, input.Parallel)
+			nodeGroupDrainer := drain.NewNodeGroupDrainer(m.clientSet, n, m.ctl.Provider.WaitTimeout(), input.MaxGracePeriod, input.NodeDrainWaitPeriod, input.PodEvictionWaitPeriod, input.Undo, input.DisableEviction, input.Parallel)
 			if err := nodeGroupDrainer.Drain(); err != nil {
 				return err
 			}

pkg/ctl/delete/cluster.go

@@ -16,12 +16,12 @@ import (
 )
 
 func deleteClusterCmd(cmd *cmdutils.Cmd) {
-	deleteClusterWithRunFunc(cmd, func(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, parallel int) error {
-		return doDeleteCluster(cmd, force, disableNodegroupEviction, parallel)
+	deleteClusterWithRunFunc(cmd, func(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, podEvictionWaitPeriod time.Duration, parallel int) error {
+		return doDeleteCluster(cmd, force, disableNodegroupEviction, podEvictionWaitPeriod, parallel)
 	})
 }
 
-func deleteClusterWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, parallel int) error) {
+func deleteClusterWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, podEvictionWaitPeriod time.Duration, parallel int) error) {
 	cfg := api.NewClusterConfig()
 	cmd.ClusterConfig = cfg
 
@@ -30,11 +30,12 @@ func deleteClusterWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cmd,
 	var (
 		force                    bool
 		disableNodegroupEviction bool
+		podEvictionWaitPeriod    time.Duration
 		parallel                 int
 	)
 	cmd.CobraCommand.RunE = func(_ *cobra.Command, args []string) error {
 		cmd.NameArg = cmdutils.GetNameArg(args)
-		return runFunc(cmd, force, disableNodegroupEviction, parallel)
+		return runFunc(cmd, force, disableNodegroupEviction, podEvictionWaitPeriod, parallel)
 	}
 
 	cmd.FlagSetGroup.InFlagSet("General", func(fs *pflag.FlagSet) {
@@ -45,6 +46,8 @@ func deleteClusterWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cmd,
 		cmdutils.AddWaitFlag(fs, &cmd.Wait, "deletion of all resources")
 		fs.BoolVar(&force, "force", false, "Force deletion to continue when errors occur")
 		fs.BoolVar(&disableNodegroupEviction, "disable-nodegroup-eviction", false, "Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution.")
+		defaultPodEvictionWaitPeriod, _ := time.ParseDuration("10s")
+		fs.DurationVar(&podEvictionWaitPeriod, "pod-eviction-wait-period", defaultPodEvictionWaitPeriod, "Duration to wait after failing to evict a pod")
 		fs.IntVar(&parallel, "parallel", 1, "Number of nodes to drain in parallel. Max 25")
 
 		cmdutils.AddConfigFileFlag(fs, &cmd.ClusterConfigFile)
@@ -54,7 +57,7 @@ func deleteClusterWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cmd,
 	cmdutils.AddCommonFlagsForAWS(cmd.FlagSetGroup, &cmd.ProviderConfig, true)
 }
 
-func doDeleteCluster(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, parallel int) error {
+func doDeleteCluster(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, podEvictionWaitPeriod time.Duration, parallel int) error {
 	if err := cmdutils.NewMetadataLoader(cmd).Load(); err != nil {
 		return err
 	}
@@ -89,5 +92,5 @@ func doDeleteCluster(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction boo
 
 	// ProviderConfig.WaitTimeout is not respected by cluster.Delete, which means the operation will never time out.
 	// When this is fixed, a deadline-based Context can be used here.
-	return cluster.Delete(context.TODO(), time.Second*20, cmd.Wait, force, disableNodegroupEviction, parallel)
+	return cluster.Delete(context.TODO(), time.Second*20, podEvictionWaitPeriod, cmd.Wait, force, disableNodegroupEviction, parallel)
 }

pkg/ctl/delete/cluster_test.go

@@ -1,6 +1,8 @@
 package delete
 
 import (
+	"time"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/ginkgo/extensions/table"
 	. "github.com/onsi/gomega"
@@ -18,7 +20,7 @@ var _ = Describe("delete cluster", func() {
 			cmd := newMockEmptyCmd(args...)
 			count := 0
 			cmdutils.AddResourceCmd(cmdutils.NewGrouping(), cmd.parentCmd, func(cmd *cmdutils.Cmd) {
-				deleteClusterWithRunFunc(cmd, func(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, parallel int) error {
+				deleteClusterWithRunFunc(cmd, func(cmd *cmdutils.Cmd, force bool, disableNodegroupEviction bool, podEvictionWaitPeriod time.Duration, parallel int) error {
 					Expect(cmd.ClusterConfig.Metadata.Name).To(Equal(clusterName))
 					Expect(force).To(Equal(forceExpected))
 					Expect(disableNodegroupEviction).To(Equal(disableNodegroupEvictionExpected))

pkg/ctl/delete/nodegroup.go

@@ -18,30 +18,31 @@ import (
 )
 
 func deleteNodeGroupCmd(cmd *cmdutils.Cmd) {
-	deleteNodeGroupWithRunFunc(cmd, func(cmd *cmdutils.Cmd, ng *api.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod time.Duration, disableEviction bool, parallel int) error {
-		return doDeleteNodeGroup(cmd, ng, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing, maxGracePeriod, disableEviction, parallel)
+	deleteNodeGroupWithRunFunc(cmd, func(cmd *cmdutils.Cmd, ng *api.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod, podEvictionWaitPeriod time.Duration, disableEviction bool, parallel int) error {
+		return doDeleteNodeGroup(cmd, ng, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing, maxGracePeriod, podEvictionWaitPeriod, disableEviction, parallel)
 	})
 }
 
-func deleteNodeGroupWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cmd, ng *api.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod time.Duration, disableEviction bool, parallel int) error) {
+func deleteNodeGroupWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cmd, ng *api.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod, podEvictionWaitPeriod time.Duration, disableEviction bool, parallel int) error) {
 	cfg := api.NewClusterConfig()
 	ng := api.NewNodeGroup()
 	cmd.ClusterConfig = cfg
 
 	var (
-		updateAuthConfigMap  bool
-		deleteNodeGroupDrain bool
-		onlyMissing          bool
-		maxGracePeriod       time.Duration
-		disableEviction      bool
-		parallel             int
+		updateAuthConfigMap   bool
+		deleteNodeGroupDrain  bool
+		onlyMissing           bool
+		maxGracePeriod        time.Duration
+		podEvictionWaitPeriod time.Duration
+		disableEviction       bool
+		parallel              int
 	)
 
 	cmd.SetDescription("nodegroup", "Delete a nodegroup", "", "ng")
 
 	cmd.CobraCommand.RunE = func(_ *cobra.Command, args []string) error {
 		cmd.NameArg = cmdutils.GetNameArg(args)
-		return runFunc(cmd, ng, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing, maxGracePeriod, disableEviction, parallel)
+		return runFunc(cmd, ng, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing, maxGracePeriod, podEvictionWaitPeriod, disableEviction, parallel)
 	}
 
 	cmd.FlagSetGroup.InFlagSet("General", func(fs *pflag.FlagSet) {
@@ -56,6 +57,8 @@ func deleteNodeGroupWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cm
 		fs.BoolVar(&deleteNodeGroupDrain, "drain", true, "Drain and cordon all nodes in the nodegroup before deletion")
 		defaultMaxGracePeriod, _ := time.ParseDuration("10m")
 		fs.DurationVar(&maxGracePeriod, "max-grace-period", defaultMaxGracePeriod, "Maximum pods termination grace period")
+		defaultPodEvictionWaitPeriod, _ := time.ParseDuration("10s")
+		fs.DurationVar(&podEvictionWaitPeriod, "pod-eviction-wait-period", defaultPodEvictionWaitPeriod, "Duration to wait after failing to evict a pod")
 		defaultDisableEviction := false
 		fs.BoolVar(&disableEviction, "disable-eviction", defaultDisableEviction, "Force drain to use delete, even if eviction is supported. This will bypass checking PodDisruptionBudgets, use with caution.")
 		fs.IntVar(&parallel, "parallel", 1, "Number of nodes to drain in parallel. Max 25")
@@ -68,7 +71,7 @@ func deleteNodeGroupWithRunFunc(cmd *cmdutils.Cmd, runFunc func(cmd *cmdutils.Cm
 	cmdutils.AddCommonFlagsForAWS(cmd.FlagSetGroup, &cmd.ProviderConfig, true)
 }
 
-func doDeleteNodeGroup(cmd *cmdutils.Cmd, ng *api.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod time.Duration, disableEviction bool, parallel int) error {
+func doDeleteNodeGroup(cmd *cmdutils.Cmd, ng *api.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod time.Duration, podEvictionWaitPeriod time.Duration, disableEviction bool, parallel int) error {
 	ngFilter := filter.NewNodeGroupFilter()
 
 	if err := cmdutils.NewDeleteAndDrainNodeGroupLoader(cmd, ng, ngFilter).Load(); err != nil {
@@ -130,11 +133,12 @@ func doDeleteNodeGroup(cmd *cmdutils.Cmd, ng *api.NodeGroup, updateAuthConfigMap
 		cmdutils.LogIntendedAction(cmd.Plan, "drain %d nodegroup(s) in cluster %q", len(allNodeGroups), cfg.Metadata.Name)
 
 		drainInput := &nodegroup.DrainInput{
-			NodeGroups:      allNodeGroups,
-			Plan:            cmd.Plan,
-			MaxGracePeriod:  maxGracePeriod,
-			DisableEviction: disableEviction,
-			Parallel:        parallel,
+			NodeGroups:            allNodeGroups,
+			Plan:                  cmd.Plan,
+			MaxGracePeriod:        maxGracePeriod,
+			PodEvictionWaitPeriod: podEvictionWaitPeriod,
+			DisableEviction:       disableEviction,
+			Parallel:              parallel,
 		}
 		err := nodeGroupManager.Drain(drainInput)
 		if err != nil {

pkg/ctl/delete/nodegroup_test.go

@@ -22,7 +22,7 @@ var _ = Describe("delete", func() {
 			cmd := newMockEmptyCmd(args...)
 			count := 0
 			cmdutils.AddResourceCmd(cmdutils.NewGrouping(), cmd.parentCmd, func(cmd *cmdutils.Cmd) {
-				deleteNodeGroupWithRunFunc(cmd, func(cmd *cmdutils.Cmd, ng *v1alpha5.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod time.Duration, disableEviction bool, parallel int) error {
+				deleteNodeGroupWithRunFunc(cmd, func(cmd *cmdutils.Cmd, ng *v1alpha5.NodeGroup, updateAuthConfigMap, deleteNodeGroupDrain, onlyMissing bool, maxGracePeriod, podEvictionWaitPeriod time.Duration, disableEviction bool, parallel int) error {
 					Expect(cmd.ClusterConfig.Metadata.Name).To(Equal("clusterName"))
 					Expect(ng.Name).To(Equal("ng"))
 					count++