diff --git a/pkg/commands/render.go b/pkg/commands/render.go index 160feb94..3a491fdb 100644 --- a/pkg/commands/render.go +++ b/pkg/commands/render.go @@ -96,8 +96,9 @@ func NewRenderSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti } renderOpts := options.Scheduler{ - Replicas: int32(commonOpts.Replicas), - PullIfNotPresent: commonOpts.PullIfNotPresent, + Replicas: int32(commonOpts.Replicas), + PullIfNotPresent: commonOpts.PullIfNotPresent, + ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, } schedObjs, err := schedManifests.Render(env.Log, renderOpts) if err != nil { diff --git a/pkg/commands/root.go b/pkg/commands/root.go index a978a58c..87bb85e5 100644 --- a/pkg/commands/root.go +++ b/pkg/commands/root.go @@ -43,8 +43,9 @@ const ( ) type internalOptions struct { - rteConfigFile string - plat string + rteConfigFile string + schedScoringStratConfigFile string + plat string } func ShowHelp(cmd *cobra.Command, args []string) error { @@ -98,6 +99,7 @@ func NewRootCommand(extraCmds ...NewCommandFunc) *cobra.Command { func InitFlags(flags *pflag.FlagSet, commonOpts *options.Options, internalOpts *internalOptions) { flags.StringVarP(&internalOpts.plat, "platform", "P", "", "platform kind:version to deploy on (example kubernetes:v1.22)") flags.StringVar(&internalOpts.rteConfigFile, "rte-config-file", "", "inject rte configuration reading from this file.") + flags.StringVar(&internalOpts.schedScoringStratConfigFile, "sched-scoring-strat-config-file", "", "inject scheduler scoring strategy configuration reading from this file.") flags.IntVarP(&commonOpts.Replicas, "replicas", "R", 1, "set the replica value - where relevant.") flags.DurationVarP(&commonOpts.WaitInterval, "wait-interval", "E", 2*time.Second, "wait interval.") @@ -142,6 +144,14 @@ func PostSetupOptions(env *deployer.Environment, commonOpts *options.Options, in commonOpts.RTEConfigData = string(data) env.Log.Info("RTE config: read", "bytes", len(commonOpts.RTEConfigData)) } + if internalOpts.schedScoringStratConfigFile != "" { + data, err := os.ReadFile(internalOpts.schedScoringStratConfigFile) + if err != nil { + return err + } + commonOpts.SchedScoringStratConfigData = string(data) + env.Log.Info("Scheduler Scoring Strategy config: read", "bytes", len(commonOpts.SchedScoringStratConfigData)) + } return validateUpdaterType(commonOpts.UpdaterType) } diff --git a/pkg/manifests/sched/sched.go b/pkg/manifests/sched/sched.go index 46aca54e..00d7408f 100644 --- a/pkg/manifests/sched/sched.go +++ b/pkg/manifests/sched/sched.go @@ -27,6 +27,7 @@ import ( apiextensionv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" "github.com/k8stopologyawareschedwg/deployer/pkg/deployer/platform" "github.com/k8stopologyawareschedwg/deployer/pkg/manifests" @@ -95,7 +96,17 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest }, } - err := schedupdate.SchedulerConfig(ret.ConfigMap, opts.ProfileName, ¶ms) + var err error + + if len(opts.ScoringStratConfigData) > 0 { + params.ScoringStrategy = &manifests.ScoringStrategyParams{} + err = yaml.Unmarshal([]byte(opts.ScoringStratConfigData), params.ScoringStrategy) + if err != nil { + return ret, err + } + } + + err = schedupdate.SchedulerConfig(ret.ConfigMap, opts.ProfileName, ¶ms) if err != nil { return ret, err } diff --git a/pkg/manifests/schedparams.go b/pkg/manifests/schedparams.go index 59488f1f..e3a14631 100644 --- a/pkg/manifests/schedparams.go +++ b/pkg/manifests/schedparams.go @@ -47,6 +47,12 @@ const ( CacheInformerDedicated = "Dedicated" ) +const ( + ScoringStrategyMostAllocated = "MostAllocated" + ScoringStrategyBalancedAllocation = "BalancedAllocation" + ScoringStrategyLeastAllocated = "LeastAllocated" +) + func ValidateForeignPodsDetectMode(value string) error { switch value { case ForeignPodsDetectNone: @@ -91,9 +97,35 @@ type ConfigCacheParams struct { InformerMode *string } +type ResourceSpecParams struct { + // Name of the resource. + Name string `json:"name"` + // Weight of the resource. + Weight int64 `json:"weight,omitempty"` +} + +type ScoringStrategyParams struct { + Type string `json:"type,omitempty"` + Resources []ResourceSpecParams `json:"resources,omitempty"` +} + +func ValidateScoringStrategyType(value string) error { + switch value { + case ScoringStrategyMostAllocated: + return nil + case ScoringStrategyBalancedAllocation: + return nil + case ScoringStrategyLeastAllocated: + return nil + default: + return fmt.Errorf("unsupported scoringStrategyType: %v", value) + } +} + type ConfigParams struct { - ProfileName string // can't be empty, so no need for pointer - Cache *ConfigCacheParams + ProfileName string // can't be empty, so no need for pointer + Cache *ConfigCacheParams + ScoringStrategy *ScoringStrategyParams } func DecodeSchedulerProfilesFromData(data []byte) ([]ConfigParams, error) { @@ -131,7 +163,7 @@ func DecodeSchedulerProfilesFromData(data []byte) ([]ConfigParams, error) { for _, plConf := range pluginConfigs { pluginConf, ok := plConf.(map[string]interface{}) if !ok { - klog.V(1).InfoS("unexpected profile coonfig data") + klog.V(1).InfoS("unexpected profile config data") return params, nil } @@ -225,5 +257,55 @@ func extractParams(profileName string, args map[string]interface{}) (ConfigParam params.Cache.InformerMode = &informerMode } } + + scoringStratArgs, ok, err := unstructured.NestedMap(args, "scoringStrategy") + if err != nil { + return params, fmt.Errorf("cannot process field scoringStrategy: %w", err) + } + if ok { + params.ScoringStrategy = &ScoringStrategyParams{} + + scoringType, cacheOk, err := unstructured.NestedString(scoringStratArgs, "type") + if err != nil { + return params, fmt.Errorf("cannot process field scoringStrategy.type: %w", err) + } + if cacheOk { + if err := ValidateScoringStrategyType(scoringType); err != nil { + return params, err + } + params.ScoringStrategy.Type = scoringType + } + + scoringRess, cacheOk, err := unstructured.NestedSlice(scoringStratArgs, "resources") + if err != nil { + return params, fmt.Errorf("cannot process field scoringStrategy.resources: %w", err) + } + if cacheOk { + var resources []ResourceSpecParams + for idx, scRes := range scoringRess { + res, ok := scRes.(map[string]interface{}) + if !ok { + return params, fmt.Errorf("unexpected scoringStrategy.resources[%d] data", idx) + } + + name, ok, err := unstructured.NestedString(res, "name") + if !ok || err != nil { + return params, fmt.Errorf("unexpected scoringStrategy.resources[%d].name data (err=%v)", idx, err) + } + + weight, ok, err := unstructured.NestedFloat64(res, "weight") + if !ok || err != nil { + return params, fmt.Errorf("unexpected scoringStrategy.resources[%d].weight data (err=%v)", idx, err) + } + + resources = append(resources, ResourceSpecParams{ + Name: name, + Weight: int64(weight), + }) + } + params.ScoringStrategy.Resources = resources + } + } + return params, nil } diff --git a/pkg/manifests/schedparams_test.go b/pkg/manifests/schedparams_test.go index 56e148ac..64ae2558 100644 --- a/pkg/manifests/schedparams_test.go +++ b/pkg/manifests/schedparams_test.go @@ -289,6 +289,197 @@ profiles: }, expectedFound: true, }, + { + name: "all scoringStrategy params", + data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + scoringStrategy: + type: MostAllocated + resources: + - name: cpu + weight: 10 + - name: memory + weight: 5 + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: topology-aware-scheduler +`), + schedulerName: "topology-aware-scheduler", + expectedParams: ConfigParams{ + ProfileName: "topology-aware-scheduler", + Cache: &ConfigCacheParams{}, + ScoringStrategy: &ScoringStrategyParams{ + Type: "MostAllocated", + Resources: []ResourceSpecParams{ + { + Name: "cpu", + Weight: int64(10), + }, + { + Name: "memory", + Weight: int64(5), + }, + }, + }, + }, + expectedFound: true, + }, + { + name: "some scoringStrategy params - 1", + data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + scoringStrategy: + type: BalancedAllocation + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: topology-aware-scheduler +`), + schedulerName: "topology-aware-scheduler", + expectedParams: ConfigParams{ + ProfileName: "topology-aware-scheduler", + Cache: &ConfigCacheParams{}, + ScoringStrategy: &ScoringStrategyParams{ + Type: "BalancedAllocation", + }, + }, + expectedFound: true, + }, + { + name: "some scoringStrategy params - 2", + data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + scoringStrategy: + resources: + - name: device.io/foobar + weight: 100 + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: topology-aware-scheduler +`), + schedulerName: "topology-aware-scheduler", + expectedParams: ConfigParams{ + ProfileName: "topology-aware-scheduler", + Cache: &ConfigCacheParams{}, + ScoringStrategy: &ScoringStrategyParams{ + Resources: []ResourceSpecParams{ + { + Name: "device.io/foobar", + Weight: int64(100), + }, + }, + }, + }, + expectedFound: true, + }, + + // keep this the last one + { + name: "nonzero resync period all cache params all scoringStrategyParams", + data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + cache: + foreignPodsDetect: None + resyncMethod: Autodetect + informerMode: Dedicated + cacheResyncPeriodSeconds: 5 + scoringStrategy: + type: BalancedAllocation + resources: + - name: cpu + weight: 10 + - name: memory + weight: 5 + - name: device.io/foobar + weight: 20 + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: topology-aware-scheduler +`), + schedulerName: "topology-aware-scheduler", + expectedParams: ConfigParams{ + ProfileName: "topology-aware-scheduler", + Cache: &ConfigCacheParams{ + ResyncPeriodSeconds: newInt64(5), + ResyncMethod: newString("Autodetect"), + ForeignPodsDetectMode: newString("None"), + InformerMode: newString("Dedicated"), + }, + ScoringStrategy: &ScoringStrategyParams{ + Type: "BalancedAllocation", + Resources: []ResourceSpecParams{ + { + Name: "cpu", + Weight: int64(10), + }, + { + Name: "memory", + Weight: int64(5), + }, + { + Name: "device.io/foobar", + Weight: int64(20), + }, + }, + }, + }, + expectedFound: true, + }, } for _, tc := range testCases { @@ -310,14 +501,14 @@ profiles: } if !reflect.DeepEqual(params, &tc.expectedParams) { - t.Fatalf("params got %q expected %q", toJSON(params), toJSON(tc.expectedParams)) + t.Fatalf("params got %s expected %s", toJSON(params), toJSON(tc.expectedParams)) } }) } } func toJSON(v any) string { - data, err := json.Marshal(v) + data, err := json.MarshalIndent(v, "", " ") if err != nil { return fmt.Sprintf("", err) } diff --git a/pkg/objectupdate/sched/render.go b/pkg/objectupdate/sched/render.go index 86e316da..0f5d2611 100644 --- a/pkg/objectupdate/sched/render.go +++ b/pkg/objectupdate/sched/render.go @@ -190,6 +190,30 @@ func updateArgs(args map[string]interface{}, params *manifests.ConfigParams) (bo return updated > 0, err } } + + scoringStratArgs, ok, err := unstructured.NestedMap(args, "scoringStrategy") + if !ok { + scoringStratArgs = make(map[string]interface{}) + } + if err != nil { + return updated > 0, err + } + + var scoringStratArgsUpdated int + if params.ScoringStrategy != nil { + scoringStratArgsUpdated, err = updateScoringStrategyArgs(scoringStratArgs, params) + if err != nil { + return updated > 0, err + } + } + updated += scoringStratArgsUpdated + + if scoringStratArgsUpdated > 0 { + if err := unstructured.SetNestedMap(args, scoringStratArgs, "scoringStrategy"); err != nil { + return updated > 0, err + } + } + return updated > 0, ensureBackwardCompatibility(args) } @@ -237,6 +261,40 @@ func updateCacheArgs(args map[string]interface{}, params *manifests.ConfigParams return updated, nil } +func updateScoringStrategyArgs(args map[string]interface{}, params *manifests.ConfigParams) (int, error) { + var updated int + var err error + + if params.ScoringStrategy.Type != "" { + scoringStratType := params.ScoringStrategy.Type // shortcut + err = manifests.ValidateScoringStrategyType(scoringStratType) + if err != nil { + return updated, err + } + err = unstructured.SetNestedField(args, scoringStratType, "type") + if err != nil { + return updated, err + } + updated++ + } + + if len(params.ScoringStrategy.Resources) > 0 { + var resources []interface{} + for _, scRes := range params.ScoringStrategy.Resources { + resources = append(resources, map[string]interface{}{ + "name": scRes.Name, + "weight": scRes.Weight, + }) + } + if err := unstructured.SetNestedSlice(args, resources, "resources"); err != nil { + return updated, err + } + updated++ + } + + return updated, nil +} + func ensureBackwardCompatibility(args map[string]interface{}) error { resyncPeriod, ok, err := unstructured.NestedInt64(args, "cacheResyncPeriodSeconds") if !ok { diff --git a/pkg/objectupdate/sched/render_test.go b/pkg/objectupdate/sched/render_test.go index 25c5108d..dd23d336 100644 --- a/pkg/objectupdate/sched/render_test.go +++ b/pkg/objectupdate/sched/render_test.go @@ -392,6 +392,137 @@ profiles: enabled: - name: NodeResourceTopologyMatch schedulerName: test-sched-name +`, + expectedUpdate: true, + }, + { + name: "partial scoring strategy params updated from empty", + params: &manifests.ConfigParams{ + ScoringStrategy: &manifests.ScoringStrategyParams{ + Type: manifests.ScoringStrategyBalancedAllocation, + }, + }, + initial: configTemplateEmpty, + expected: `apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + scoringStrategy: + type: BalancedAllocation + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: test-sched-name +`, + expectedUpdate: true, + }, + { + name: "partial scoring strategy params updated from empty - 2", + params: &manifests.ConfigParams{ + ScoringStrategy: &manifests.ScoringStrategyParams{ + Type: manifests.ScoringStrategyBalancedAllocation, + Resources: []manifests.ResourceSpecParams{ + { + Name: "cpu", + Weight: int64(20), + }, + { + Name: "fancy.com/device", + Weight: int64(100), + }, + }, + }, + }, + initial: configTemplateEmpty, + expected: `apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + scoringStrategy: + resources: + - name: cpu + weight: 20 + - name: fancy.com/device + weight: 100 + type: BalancedAllocation + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: test-sched-name +`, + expectedUpdate: true, + }, + { + name: "partial scoring strategy params updated from nonempty", + params: &manifests.ConfigParams{ + ScoringStrategy: &manifests.ScoringStrategyParams{ + Type: manifests.ScoringStrategyBalancedAllocation, + Resources: []manifests.ResourceSpecParams{ + { + Name: "cpu", + Weight: int64(20), + }, + { + Name: "fancy.com/device", + Weight: int64(100), + }, + }, + }, + }, + initial: configTemplateAllValuesScoringFineTuned, + expected: `apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + cache: + foreignPodsDetect: OnlyExclusiveResources + informerMode: Dedicated + resyncMethod: OnlyExclusiveResources + cacheResyncPeriodSeconds: 5 + scoringStrategy: + resources: + - name: cpu + weight: 20 + - name: fancy.com/device + weight: 100 + type: BalancedAllocation + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: test-sched-name `, expectedUpdate: true, }, @@ -519,6 +650,37 @@ profiles: schedulerName: test-sched-name ` +var configTemplateAllValuesScoringFineTuned string = `apiVersion: kubescheduler.config.k8s.io/v1beta3 +kind: KubeSchedulerConfiguration +leaderElection: + leaderElect: false +profiles: +- pluginConfig: + - args: + cache: + foreignPodsDetect: OnlyExclusiveResources + informerMode: Dedicated + resyncMethod: OnlyExclusiveResources + cacheResyncPeriodSeconds: 5 + scoringStrategy: + resources: + - name: cpu + weight: 2 + type: MostAllocated + name: NodeResourceTopologyMatch + plugins: + filter: + enabled: + - name: NodeResourceTopologyMatch + reserve: + enabled: + - name: NodeResourceTopologyMatch + score: + enabled: + - name: NodeResourceTopologyMatch + schedulerName: test-sched-name +` + var configTemplateAllValuesMultiRenamed string = `apiVersion: kubescheduler.config.k8s.io/v1beta3 kind: KubeSchedulerConfiguration leaderElection: diff --git a/pkg/options/options.go b/pkg/options/options.go index 2994430f..c2408676 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -25,26 +25,27 @@ import ( ) type Options struct { - UserPlatform platform.Platform - UserPlatformVersion platform.Version - Replicas int - RTEConfigData string - PullIfNotPresent bool - UpdaterType string - UpdaterPFPEnable bool - UpdaterNotifEnable bool - UpdaterCRIHooksEnable bool - UpdaterSyncPeriod time.Duration - UpdaterVerbose int - SchedProfileName string - SchedResyncPeriod time.Duration - SchedVerbose int - SchedCtrlPlaneAffinity bool - WaitInterval time.Duration - WaitTimeout time.Duration - ClusterPlatform platform.Platform - ClusterVersion platform.Version - WaitCompletion bool + UserPlatform platform.Platform + UserPlatformVersion platform.Version + Replicas int + RTEConfigData string + PullIfNotPresent bool + UpdaterType string + UpdaterPFPEnable bool + UpdaterNotifEnable bool + UpdaterCRIHooksEnable bool + UpdaterSyncPeriod time.Duration + UpdaterVerbose int + SchedProfileName string + SchedResyncPeriod time.Duration + SchedVerbose int + SchedCtrlPlaneAffinity bool + WaitInterval time.Duration + WaitTimeout time.Duration + ClusterPlatform platform.Platform + ClusterVersion platform.Version + WaitCompletion bool + SchedScoringStratConfigData string } type API struct { @@ -52,14 +53,15 @@ type API struct { } type Scheduler struct { - Platform platform.Platform - WaitCompletion bool - Replicas int32 - ProfileName string - PullIfNotPresent bool - CacheResyncPeriod time.Duration - CtrlPlaneAffinity bool - Verbose int + Platform platform.Platform + WaitCompletion bool + Replicas int32 + ProfileName string + PullIfNotPresent bool + CacheResyncPeriod time.Duration + CtrlPlaneAffinity bool + Verbose int + ScoringStratConfigData string } type DaemonSet struct {