From f725f725c2368166ba27b78fdbbee006849b32e9 Mon Sep 17 00:00:00 2001 From: Alex Marston Date: Fri, 15 Mar 2024 12:11:23 +0000 Subject: [PATCH] Validate Karpenter Disruption taints as part of preStop hook --- cmd/hooks/prestop.go | 5 ++ cmd/hooks/prestop_test.go | 153 ++++++++++++++++++++++++++++++++++++++ go.mod | 7 +- go.sum | 12 +++ 4 files changed, 176 insertions(+), 1 deletion(-) diff --git a/cmd/hooks/prestop.go b/cmd/hooks/prestop.go index 72e35a1134..3c93f8a70c 100644 --- a/cmd/hooks/prestop.go +++ b/cmd/hooks/prestop.go @@ -12,6 +12,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) /* @@ -62,6 +63,10 @@ func isNodeBeingDrained(node *v1.Node) bool { if taint.Key == v1.TaintNodeUnschedulable && taint.Effect == v1.TaintEffectNoSchedule { return true } + + if corev1beta1.IsDisruptingTaint(taint) { + return true + } } return false } diff --git a/cmd/hooks/prestop_test.go b/cmd/hooks/prestop_test.go index f9cb6ae704..ab843933c2 100644 --- a/cmd/hooks/prestop_test.go +++ b/cmd/hooks/prestop_test.go @@ -10,6 +10,7 @@ import ( v1 "k8s.io/api/core/v1" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/watch" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) func TestPreStopHook(t *testing.T) { @@ -209,6 +210,158 @@ func TestPreStopHook(t *testing.T) { return nil }, }, + { + name: "TestPreStopHook: Karpenter node is not being drained, skipping VolumeAttachments check - missing TaintEffectNoSchedule", + nodeName: "test-karpenter-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockStorageV1 *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + mockNodeObj := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: corev1beta1.DisruptionNoScheduleTaint.Key, + Effect: "", + }, + }, + }, + } + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).Times(1) + mockCoreV1.EXPECT().Nodes().Return(mockNode).Times(1) + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(mockNodeObj, nil).Times(1) + + return nil + }, + }, + { + name: "TestPreStopHook: Karpenter node is being drained, no volume attachments remain", + nodeName: "test-karpenter-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: corev1beta1.DisruptionNoScheduleTaint.Key, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + emptyVolumeAttachments := &storagev1.VolumeAttachmentList{Items: []storagev1.VolumeAttachment{}} + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(emptyVolumeAttachments, nil).AnyTimes() + mockVolumeAttachments.EXPECT().Watch(gomock.Any(), gomock.Any()).Return(watch.NewFake(), nil).AnyTimes() + + return nil + }, + }, + { + name: "TestPreStopHook: Karpenter node is being drained, no volume attachments associated with node", + nodeName: "test-karpenter-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: corev1beta1.DisruptionNoScheduleTaint.Key, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + fakeVolumeAttachments := &storagev1.VolumeAttachmentList{ + Items: []storagev1.VolumeAttachment{ + { + Spec: storagev1.VolumeAttachmentSpec{ + NodeName: "test-node-2", + }, + }, + }, + } + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(fakeVolumeAttachments, nil).AnyTimes() + mockVolumeAttachments.EXPECT().Watch(gomock.Any(), gomock.Any()).Return(watch.NewFake(), nil).AnyTimes() + + return nil + }, + }, + { + name: "TestPreStopHook: Karpenter Node is drained before timeout", + nodeName: "test-karpenter-node", + expErr: nil, + mockFunc: func(nodeName string, mockClient *driver.MockKubernetesClient, mockCoreV1 *driver.MockCoreV1Interface, mockNode *driver.MockNodeInterface, mockVolumeAttachments *driver.MockVolumeAttachmentInterface, mockStorageV1Interface *driver.MockStorageV1Interface) error { + + fakeNode := &v1.Node{ + Spec: v1.NodeSpec{ + Taints: []v1.Taint{ + { + Key: corev1beta1.DisruptionNoScheduleTaint.Key, + Effect: v1.TaintEffectNoSchedule, + }, + }, + }, + } + + fakeVolumeAttachments := &storagev1.VolumeAttachmentList{ + Items: []storagev1.VolumeAttachment{ + { + Spec: storagev1.VolumeAttachmentSpec{ + NodeName: "test-karpenter-node", + }, + }, + }, + } + + fakeWatcher := watch.NewFake() + deleteSignal := make(chan bool, 1) + + mockClient.EXPECT().CoreV1().Return(mockCoreV1).AnyTimes() + mockClient.EXPECT().StorageV1().Return(mockStorageV1Interface).AnyTimes() + + mockCoreV1.EXPECT().Nodes().Return(mockNode).AnyTimes() + mockNode.EXPECT().Get(gomock.Any(), gomock.Eq(nodeName), gomock.Any()).Return(fakeNode, nil).AnyTimes() + + mockStorageV1Interface.EXPECT().VolumeAttachments().Return(mockVolumeAttachments).AnyTimes() + gomock.InOrder( + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(fakeVolumeAttachments, nil).AnyTimes(), + mockVolumeAttachments.EXPECT().Watch(gomock.Any(), gomock.Any()).DoAndReturn(func(signal, watchSignal interface{}) (watch.Interface, error) { + deleteSignal <- true + return fakeWatcher, nil + }).AnyTimes(), + mockVolumeAttachments.EXPECT().List(gomock.Any(), gomock.Any()).Return(&storagev1.VolumeAttachmentList{Items: []storagev1.VolumeAttachment{}}, nil).AnyTimes(), + ) + + go func() { + <-deleteSignal + fakeWatcher.Delete(&storagev1.VolumeAttachment{ + Spec: storagev1.VolumeAttachmentSpec{ + NodeName: "test-karpenter-node", + }, + }) + }() + return nil + }, + }, } for _, tc := range testCases { diff --git a/go.mod b/go.mod index 37be241dea..14c39a345e 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,11 @@ require ( github.com/antlr/antlr4/runtime/Go/antlr/v4 v4.0.0-20230305170008-8188dc5388df // indirect github.com/distribution/reference v0.5.0 // indirect github.com/gorilla/websocket v1.5.1 // indirect + github.com/mitchellh/hashstructure/v2 v2.0.2 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/samber/lo v1.39.0 // indirect + knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4 // indirect ) require ( @@ -102,7 +106,7 @@ require ( go.opentelemetry.io/otel/trace v1.23.1 // indirect go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect + go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.19.0 // indirect golang.org/x/exp v0.0.0-20240213143201-ec583247a57a // indirect golang.org/x/mod v0.15.0 // indirect @@ -133,6 +137,7 @@ require ( k8s.io/kubelet v0.29.2 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/karpenter v0.35.2 sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index 1e0b77a471..93f30b6d6a 100644 --- a/go.sum +++ b/go.sum @@ -956,6 +956,8 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= +github.com/mitchellh/hashstructure/v2 v2.0.2/go.mod h1:MG3aRVU/N29oo/V/IhBX8GR/zz4kQkprJgF2EVszyDE= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/moby/sys/mountinfo v0.7.1 h1:/tTvQaSJRr2FshkhXiIpux6fQ2Zvc4j7tAhMTStAG2g= @@ -1044,6 +1046,8 @@ github.com/prometheus/common v0.47.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5E github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -1054,6 +1058,8 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= +github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= +github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -1149,6 +1155,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -1959,6 +1967,8 @@ k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/ k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4 h1:oO/BQJpVCFTSTMHF/S6u+nPtIvbHDTsvbPZvdCZAFjs= +knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4/go.mod h1:eXobTqst4aI7CNa6W7sG73VhEsHGWPSrkefeMTb++a0= lukechampine.com/uint128 v1.1.1/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.36.0/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= @@ -2001,6 +2011,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 h1:/U5vjBbQn3RCh sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0/go.mod h1:z7+wmGM2dfIiLRfrC6jb5kV2Mq/sK1ZP303cxzkV5Y4= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/karpenter v0.35.2 h1:Q81+OLvMrLIrV9ueV14YFiDBuS348uxBTCIITFAA0fw= +sigs.k8s.io/karpenter v0.35.2/go.mod h1:cUrYd9r91yK9Y4k6N87kdhs0EYTAlp9yimSqFcckxAw= sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=