From b6f4ef09140f74d6043475353c94d742bbd0d02b Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Fri, 19 Jan 2024 14:40:42 +0100 Subject: [PATCH] sched: support for informerMode option The scheduler plugin gained support for the external informer, necessary to overcome mismatch between the scheduler and the ode views wrt the pods on nodes. Expose support in the deployer libraries to change this setting. Signed-off-by: Francesco Romani --- pkg/manifests/schedparams.go | 28 ++++++++++++ pkg/manifests/schedparams_test.go | 37 ++++++++++++++- pkg/objectupdate/sched/render.go | 65 ++++++++++++++++----------- pkg/objectupdate/sched/render_test.go | 9 +++- 4 files changed, 111 insertions(+), 28 deletions(-) diff --git a/pkg/manifests/schedparams.go b/pkg/manifests/schedparams.go index 047c6388..59488f1f 100644 --- a/pkg/manifests/schedparams.go +++ b/pkg/manifests/schedparams.go @@ -42,6 +42,11 @@ const ( CacheResyncOnlyExclusiveResources = "OnlyExclusiveResources" ) +const ( + CacheInformerShared = "Shared" + CacheInformerDedicated = "Dedicated" +) + func ValidateForeignPodsDetectMode(value string) error { switch value { case ForeignPodsDetectNone: @@ -68,10 +73,22 @@ func ValidateCacheResyncMethod(value string) error { } } +func ValidateCacheInformerMode(value string) error { + switch value { + case CacheInformerShared: + return nil + case CacheInformerDedicated: + return nil + default: + return fmt.Errorf("unsupported cacheInformerMode: %v", value) + } +} + type ConfigCacheParams struct { ResyncPeriodSeconds *int64 ResyncMethod *string ForeignPodsDetectMode *string + InformerMode *string } type ConfigParams struct { @@ -196,6 +213,17 @@ func extractParams(profileName string, args map[string]interface{}) (ConfigParam } params.Cache.ForeignPodsDetectMode = &foreignPodsDetect } + + informerMode, cacheOk, err := unstructured.NestedString(cacheArgs, "informerMode") + if err != nil { + return params, fmt.Errorf("cannot process field cache.informerMode: %w", err) + } + if cacheOk { + if err := ValidateCacheInformerMode(informerMode); err != nil { + return params, err + } + params.Cache.InformerMode = &informerMode + } } return params, nil } diff --git a/pkg/manifests/schedparams_test.go b/pkg/manifests/schedparams_test.go index bc0baacd..56e148ac 100644 --- a/pkg/manifests/schedparams_test.go +++ b/pkg/manifests/schedparams_test.go @@ -161,6 +161,7 @@ profiles: cache: foreignPodsDetect: None resyncMethod: Autodetect + informerMode: Dedicated cacheResyncPeriodSeconds: 5 name: NodeResourceTopologyMatch plugins: @@ -182,6 +183,7 @@ profiles: ResyncPeriodSeconds: newInt64(5), ResyncMethod: newString("Autodetect"), ForeignPodsDetectMode: newString("None"), + InformerMode: newString("Dedicated"), }, }, expectedFound: true, @@ -222,7 +224,7 @@ profiles: expectedFound: true, }, { - name: "zero resync period and some cache params", + name: "zero resync period and some cache params - 2", data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration leaderElection: @@ -254,6 +256,39 @@ profiles: }, expectedFound: true, }, + { + name: "zero resync period and some cache params - 3", + data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + cache: + informerMode: Shared + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: topology-aware-scheduler +`), + schedulerName: "topology-aware-scheduler", + expectedParams: ConfigParams{ + ProfileName: "topology-aware-scheduler", + Cache: &ConfigCacheParams{ + InformerMode: newString("Shared"), + }, + }, + expectedFound: true, + }, } for _, tc := range testCases { diff --git a/pkg/objectupdate/sched/render.go b/pkg/objectupdate/sched/render.go index 390fb871..86e316da 100644 --- a/pkg/objectupdate/sched/render.go +++ b/pkg/objectupdate/sched/render.go @@ -176,9 +176,12 @@ func updateArgs(args map[string]interface{}, params *manifests.ConfigParams) (bo return updated > 0, err } - cacheArgsUpdated, err := updateCacheArgs(cacheArgs, params) - if err != nil { - return updated > 0, err + var cacheArgsUpdated int + if params.Cache != nil { + cacheArgsUpdated, err = updateCacheArgs(cacheArgs, params) + if err != nil { + return updated > 0, err + } } updated += cacheArgsUpdated @@ -194,31 +197,41 @@ func updateCacheArgs(args map[string]interface{}, params *manifests.ConfigParams var updated int var err error - if params.Cache != nil { - if params.Cache.ResyncMethod != nil { - resyncMethod := *params.Cache.ResyncMethod // shortcut - err = manifests.ValidateCacheResyncMethod(resyncMethod) - if err != nil { - return updated, err - } - err = unstructured.SetNestedField(args, resyncMethod, "resyncMethod") - if err != nil { - return updated, err - } - updated++ + if params.Cache.ResyncMethod != nil { + resyncMethod := *params.Cache.ResyncMethod // shortcut + err = manifests.ValidateCacheResyncMethod(resyncMethod) + if err != nil { + return updated, err } - if params.Cache.ForeignPodsDetectMode != nil { - foreignPodsMode := *params.Cache.ForeignPodsDetectMode // shortcut - err = manifests.ValidateForeignPodsDetectMode(foreignPodsMode) - if err != nil { - return updated, err - } - err = unstructured.SetNestedField(args, foreignPodsMode, "foreignPodsDetect") - if err != nil { - return updated, err - } - updated++ + err = unstructured.SetNestedField(args, resyncMethod, "resyncMethod") + if err != nil { + return updated, err + } + updated++ + } + if params.Cache.ForeignPodsDetectMode != nil { + foreignPodsMode := *params.Cache.ForeignPodsDetectMode // shortcut + err = manifests.ValidateForeignPodsDetectMode(foreignPodsMode) + if err != nil { + return updated, err + } + err = unstructured.SetNestedField(args, foreignPodsMode, "foreignPodsDetect") + if err != nil { + return updated, err + } + updated++ + } + if params.Cache.InformerMode != nil { + informerMode := *params.Cache.InformerMode // shortcut + err = manifests.ValidateCacheInformerMode(informerMode) + if err != nil { + return updated, err + } + err = unstructured.SetNestedField(args, informerMode, "informerMode") + if err != nil { + return updated, err } + updated++ } return updated, nil diff --git a/pkg/objectupdate/sched/render_test.go b/pkg/objectupdate/sched/render_test.go index 27fe663a..25c5108d 100644 --- a/pkg/objectupdate/sched/render_test.go +++ b/pkg/objectupdate/sched/render_test.go @@ -19,6 +19,7 @@ package sched import ( "testing" + "github.com/google/go-cmp/cmp" "github.com/k8stopologyawareschedwg/deployer/pkg/manifests" ) @@ -220,6 +221,7 @@ profiles: ResyncPeriodSeconds: newInt64(42), ResyncMethod: newString("OnlyExclusiveResources"), ForeignPodsDetectMode: newString("OnlyExclusiveResources"), + InformerMode: newString("Dedicated"), }, }, initial: configTemplateEmpty, @@ -232,6 +234,7 @@ profiles: - args: cache: foreignPodsDetect: OnlyExclusiveResources + informerMode: Dedicated resyncMethod: OnlyExclusiveResources cacheResyncPeriodSeconds: 42 name: NodeResourceTopologyMatch @@ -292,6 +295,7 @@ profiles: ResyncPeriodSeconds: newInt64(7), ResyncMethod: newString("Autodetect"), ForeignPodsDetectMode: newString("None"), + InformerMode: newString("Shared"), }, }, initial: configTemplateAllValuesFineTuned, @@ -304,6 +308,7 @@ profiles: - args: cache: foreignPodsDetect: None + informerMode: Shared resyncMethod: Autodetect cacheResyncPeriodSeconds: 7 name: NodeResourceTopologyMatch @@ -372,6 +377,7 @@ profiles: - args: cache: foreignPodsDetect: All + informerMode: Dedicated resyncMethod: OnlyExclusiveResources cacheResyncPeriodSeconds: 5 name: NodeResourceTopologyMatch @@ -409,7 +415,7 @@ profiles: rendered := string(data) if rendered != tc.expected { - t.Errorf("rendering failed.\nrendered=[%s]\nexpected=[%s]\n", rendered, tc.expected) + t.Errorf("rendering failed.\nrendered=[%s]\nexpected=[%s]\ndiff=[%s]\n", rendered, tc.expected, cmp.Diff(rendered, tc.expected)) } }) } @@ -467,6 +473,7 @@ profiles: - args: cache: foreignPodsDetect: OnlyExclusiveResources + informerMode: Dedicated resyncMethod: OnlyExclusiveResources cacheResyncPeriodSeconds: 5 name: NodeResourceTopologyMatch