From 509fd9962931bcdbd85c4544c8772485e920fbb0 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Sat, 9 Dec 2023 21:53:29 -0800 Subject: [PATCH 1/2] Avoid unnecessary requeue operations in coscheduling --- pkg/coscheduling/core/core.go | 34 ++++++++++++++++++++++++++++-- pkg/coscheduling/core/core_test.go | 3 ++- pkg/coscheduling/coscheduling.go | 2 +- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pkg/coscheduling/core/core.go b/pkg/coscheduling/core/core.go index fd9decdca..1d3e52163 100644 --- a/pkg/coscheduling/core/core.go +++ b/pkg/coscheduling/core/core.go @@ -48,12 +48,22 @@ const ( PodGroupNotFound Status = "PodGroup not found" Success Status = "Success" Wait Status = "Wait" + + preFilterStateKey = "PreFilterCoscheduling" ) +type PreFilterState struct { + Activate bool +} + +func (s *PreFilterState) Clone() framework.StateData { + return &PreFilterState{Activate: s.Activate} +} + // Manager defines the interfaces for PodGroup management. type Manager interface { PreFilter(context.Context, *corev1.Pod) error - Permit(context.Context, *corev1.Pod) Status + Permit(context.Context, *framework.CycleState, *corev1.Pod) Status GetPodGroup(context.Context, *corev1.Pod) (string, *v1alpha1.PodGroup) GetCreationTimestamp(*corev1.Pod, time.Time) time.Time DeletePermittedPodGroup(string) @@ -108,6 +118,13 @@ func (pgMgr *PodGroupManager) ActivateSiblings(pod *corev1.Pod, state *framework return } + // Only proceed if it's explicitly requested to activate sibling pods. + if c, err := state.Read(preFilterStateKey); err != nil { + return + } else if s, ok := c.(*PreFilterState); !ok || !s.Activate { + return + } + pods, err := pgMgr.podLister.Pods(pod.Namespace).List( labels.SelectorFromSet(labels.Set{v1alpha1.PodGroupLabel: pgName}), ) @@ -193,7 +210,7 @@ func (pgMgr *PodGroupManager) PreFilter(ctx context.Context, pod *corev1.Pod) er } // Permit permits a pod to run, if the minMember match, it would send a signal to chan. -func (pgMgr *PodGroupManager) Permit(ctx context.Context, pod *corev1.Pod) Status { +func (pgMgr *PodGroupManager) Permit(ctx context.Context, state *framework.CycleState, pod *corev1.Pod) Status { pgFullName, pg := pgMgr.GetPodGroup(ctx, pod) if pgFullName == "" { return PodGroupNotSpecified @@ -209,6 +226,19 @@ func (pgMgr *PodGroupManager) Permit(ctx context.Context, pod *corev1.Pod) Statu if int32(assigned)+1 >= pg.Spec.MinMember { return Success } + + if assigned == 0 { + // Given we've reached Permit(), it's mean all PreFilter checks (minMember & minResource) + // already pass through, so if assigned == 0, it could be due to: + // - minResource get satisfied + // - new pods added + // In either case, we should and only should use this 0-th pod to trigger activating + // its siblings. + // It'd be in-efficient if we trigger activating siblings unconditionally. + // See https://github.com/kubernetes-sigs/scheduler-plugins/issues/682 + state.Write(preFilterStateKey, &PreFilterState{Activate: true}) + } + return Wait } diff --git a/pkg/coscheduling/core/core_test.go b/pkg/coscheduling/core/core_test.go index c57f659c8..1918bae87 100644 --- a/pkg/coscheduling/core/core_test.go +++ b/pkg/coscheduling/core/core_test.go @@ -28,6 +28,7 @@ import ( "k8s.io/client-go/informers" clientsetfake "k8s.io/client-go/kubernetes/fake" clicache "k8s.io/client-go/tools/cache" + "k8s.io/kubernetes/pkg/scheduler/framework" st "k8s.io/kubernetes/pkg/scheduler/testing" "sigs.k8s.io/scheduler-plugins/apis/scheduling/v1alpha1" tu "sigs.k8s.io/scheduler-plugins/test/util" @@ -278,7 +279,7 @@ func TestPermit(t *testing.T) { podInformer.Informer().GetStore().Add(p) } - if got := pgMgr.Permit(ctx, tt.pod); got != tt.want { + if got := pgMgr.Permit(ctx, &framework.CycleState{}, tt.pod); got != tt.want { t.Errorf("Want %v, but got %v", tt.want, got) } }) diff --git a/pkg/coscheduling/coscheduling.go b/pkg/coscheduling/coscheduling.go index 8c61c60d5..bcf37e2fc 100644 --- a/pkg/coscheduling/coscheduling.go +++ b/pkg/coscheduling/coscheduling.go @@ -204,7 +204,7 @@ func (cs *Coscheduling) PreFilterExtensions() framework.PreFilterExtensions { // Permit is the functions invoked by the framework at "Permit" extension point. func (cs *Coscheduling) Permit(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (*framework.Status, time.Duration) { waitTime := *cs.scheduleTimeout - s := cs.pgMgr.Permit(ctx, pod) + s := cs.pgMgr.Permit(ctx, state, pod) var retStatus *framework.Status switch s { case core.PodGroupNotSpecified: From 873a591534b6f3e2a45a0b33e97ba664c43bf3a2 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 31 Jan 2024 09:28:00 -0800 Subject: [PATCH 2/2] fixup: rename preFilterState to permitState --- pkg/coscheduling/core/core.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/coscheduling/core/core.go b/pkg/coscheduling/core/core.go index 1d3e52163..60d1b584f 100644 --- a/pkg/coscheduling/core/core.go +++ b/pkg/coscheduling/core/core.go @@ -49,15 +49,15 @@ const ( Success Status = "Success" Wait Status = "Wait" - preFilterStateKey = "PreFilterCoscheduling" + permitStateKey = "PermitCoscheduling" ) -type PreFilterState struct { +type PermitState struct { Activate bool } -func (s *PreFilterState) Clone() framework.StateData { - return &PreFilterState{Activate: s.Activate} +func (s *PermitState) Clone() framework.StateData { + return &PermitState{Activate: s.Activate} } // Manager defines the interfaces for PodGroup management. @@ -119,9 +119,9 @@ func (pgMgr *PodGroupManager) ActivateSiblings(pod *corev1.Pod, state *framework } // Only proceed if it's explicitly requested to activate sibling pods. - if c, err := state.Read(preFilterStateKey); err != nil { + if c, err := state.Read(permitStateKey); err != nil { return - } else if s, ok := c.(*PreFilterState); !ok || !s.Activate { + } else if s, ok := c.(*PermitState); !ok || !s.Activate { return } @@ -236,7 +236,7 @@ func (pgMgr *PodGroupManager) Permit(ctx context.Context, state *framework.Cycle // its siblings. // It'd be in-efficient if we trigger activating siblings unconditionally. // See https://github.com/kubernetes-sigs/scheduler-plugins/issues/682 - state.Write(preFilterStateKey, &PreFilterState{Activate: true}) + state.Write(permitStateKey, &PermitState{Activate: true}) } return Wait