diff --git a/pkg/controllers/node/termination/controller.go b/pkg/controllers/node/termination/controller.go index b8e49a3931..f51bcafa05 100644 --- a/pkg/controllers/node/termination/controller.go +++ b/pkg/controllers/node/termination/controller.go @@ -28,6 +28,7 @@ import ( storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/workqueue" "k8s.io/utils/clock" @@ -193,6 +194,15 @@ func (c *Controller) ensureVolumesDetached(ctx context.Context, node *corev1.Nod return len(filteredVolumeAttachments) == 0, nil } +// volumeIsNotAttachedOrBound returns true if the persistent volume storage is neither Attached nor Bound +func volumeIsNotAttachedOrBound(ctx context.Context, kubeClient client.Client, pvName string) (bool, error) { + pv := &corev1.PersistentVolume{} + if err := kubeClient.Get(ctx, types.NamespacedName{Name: pvName}, pv); err != nil { + return false, fmt.Errorf("getting persistent volume %q, %w", pvName, err) + } + return pv.Status.Phase == corev1.VolumePending || pv.Status.Phase == corev1.VolumeReleased || pv.Status.Phase == corev1.VolumeFailed, nil +} + // filterVolumeAttachments filters out storagev1.VolumeAttachments that should not block the termination // of the passed corev1.Node func filterVolumeAttachments(ctx context.Context, kubeClient client.Client, node *corev1.Node, volumeAttachments []*storagev1.VolumeAttachment, clk clock.Clock) ([]*storagev1.VolumeAttachment, error) { @@ -225,6 +235,21 @@ func filterVolumeAttachments(ctx context.Context, kubeClient client.Client, node } } } + + // ignore volume attachments associated with volumes that are no longer bound + for _, va := range volumeAttachments { + unattached, err := volumeIsNotAttachedOrBound(ctx, kubeClient, *va.Spec.Source.PersistentVolumeName) + if errors.IsNotFound(err) { + continue + } + if err != nil { + return nil, err + } + if unattached { + shouldFilterOutVolume.Insert(*va.Spec.Source.PersistentVolumeName) + } + } + filteredVolumeAttachments := lo.Reject(volumeAttachments, func(v *storagev1.VolumeAttachment, _ int) bool { pvName := v.Spec.Source.PersistentVolumeName return pvName == nil || shouldFilterOutVolume.Has(*pvName) diff --git a/pkg/controllers/node/termination/suite_test.go b/pkg/controllers/node/termination/suite_test.go index 32151e311f..5be532e8f6 100644 --- a/pkg/controllers/node/termination/suite_test.go +++ b/pkg/controllers/node/termination/suite_test.go @@ -800,6 +800,25 @@ var _ = Describe("Termination", func() { ExpectObjectReconciled(ctx, env.Client, terminationController, node) ExpectNotFound(ctx, env.Client, node) }) + It("should not wait for volume attachments with released persistent volumes", func() { + va := test.VolumeAttachment(test.VolumeAttachmentOptions{ + NodeName: node.Name, + VolumeName: "foo", + }) + pv := test.PersistentVolume(test.PersistentVolumeOptions{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + }, + }) + pv.Status.Phase = corev1.VolumeReleased + + ExpectApplied(ctx, env.Client, node, nodeClaim, nodePool, pv, va) + Expect(env.Client.Delete(ctx, node)).To(Succeed()) + + ExpectObjectReconciled(ctx, env.Client, terminationController, node) + ExpectObjectReconciled(ctx, env.Client, terminationController, node) + ExpectNotFound(ctx, env.Client, node) + }) It("should wait for volume attachments until the nodeclaim's termination grace period expires", func() { va := test.VolumeAttachment(test.VolumeAttachmentOptions{ NodeName: node.Name,