WIP

Signed-off-by: Francesco Romani <[email protected]>
k8stopologyawareschedwg · Mar 5, 2024 · 723eaee · 723eaee
1 parent 1f994c8
commit 723eaee
Show file tree

Hide file tree

Showing 7 changed files with 87 additions and 3 deletions.
diff --git a/pkg/commands/deploy.go b/pkg/commands/deploy.go
@@ -112,6 +112,7 @@ func NewDeploySchedulerPluginCommand(env *deployer.Environment, commonOpts *opti
 				Verbose:                commonOpts.SchedVerbose,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
 			})
 		},
 		Args: cobra.NoArgs,

diff --git a/pkg/commands/remove.go b/pkg/commands/remove.go
@@ -64,6 +64,7 @@ func NewRemoveCommand(env *deployer.Environment, commonOpts *options.Options) *c
 				Verbose:                commonOpts.SchedVerbose,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
 			})
 			if err != nil {
 				// intentionally keep going to remove as much as possible
@@ -166,6 +167,7 @@ func NewRemoveSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti
 				Verbose:                commonOpts.SchedVerbose,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
 			})
 		},
 		Args: cobra.NoArgs,

diff --git a/pkg/commands/render.go b/pkg/commands/render.go
@@ -101,6 +101,7 @@ func NewRenderSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti
 				CacheResyncPeriod:      commonOpts.SchedResyncPeriod,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
 			}
 			schedObjs, err := schedManifests.Render(env.Log, renderOpts)
 			if err != nil {
@@ -186,6 +187,7 @@ func RenderManifests(env *deployer.Environment, commonOpts *options.Options) err
 		Verbose:                commonOpts.SchedVerbose,
 		ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 		CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+		LeaderElection:         commonOpts.Replicas > 1,
 	}
 
 	schedObjs, err := schedManifests.Render(env.Log, schedRenderOpts)

diff --git a/pkg/manifests/sched/sched.go b/pkg/manifests/sched/sched.go
@@ -104,16 +104,20 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest
 		Cache:       manifests.NewConfigCacheParams(),
 	}
 
+	if opts.LeaderElection {
+		leap := manifests.LeaderElectionParams{}
+		manifests.SetDefaultsLeaderElection(&leap)
+		leap.LeaderElect = true
+		params.LeaderElection = &leap
+	}
+
 	if len(opts.CacheParamsConfigData) > 0 {
 		err = yaml.Unmarshal([]byte(opts.CacheParamsConfigData), params.Cache)
 		if err != nil {
 			return ret, err
 		}
 	}
 
-	// always override
-	params.Cache.ResyncPeriodSeconds = newInt64(int64(opts.CacheResyncPeriod.Seconds()))
-
 	if len(opts.ScoringStratConfigData) > 0 {
 		params.ScoringStrategy = &manifests.ScoringStrategyParams{}
 		err = yaml.Unmarshal([]byte(opts.ScoringStratConfigData), params.ScoringStrategy)
@@ -122,6 +126,9 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest
 		}
 	}
 
+	// always override
+	params.Cache.ResyncPeriodSeconds = newInt64(int64(opts.CacheResyncPeriod.Seconds()))
+
 	err = schedupdate.SchedulerConfig(ret.ConfigMap, DefaultProfileName, &params)
 	if err != nil {
 		return ret, err

diff --git a/pkg/manifests/schedparams.go b/pkg/manifests/schedparams.go
@@ -54,6 +54,11 @@ const (
 	ScoringStrategyLeastAllocated     = "LeastAllocated"
 )
 
+const (
+	LeaderElectionDefaultName      = "k8stawwg-nrtmatch-scheduler"
+	LeaderElectionDefaultNamespace = "k8stawwg"
+)
+
 func ValidateForeignPodsDetectMode(value string) error {
 	switch value {
 	case ForeignPodsDetectNone:
@@ -136,10 +141,26 @@ func ValidateScoringStrategyType(value string) error {
 	}
 }
 
+type LeaderElectionParams struct {
+	LeaderElect       bool   `json:"leaderElect,omitempty"`
+	ResourceName      string `json:"resourceName,omitempty"`
+	ResourceNamespace string `json:"resourceNamespace,omitempty"`
+}
+
+func SetDefaultsLeaderElection(lep *LeaderElectionParams) {
+	if lep.ResourceName == "" {
+		lep.ResourceName = LeaderElectionDefaultName
+	}
+	if lep.ResourceNamespace == "" {
+		lep.ResourceNamespace = LeaderElectionDefaultNamespace
+	}
+}
+
 type ConfigParams struct {
 	ProfileName     string // can't be empty, so no need for pointer
 	Cache           *ConfigCacheParams
 	ScoringStrategy *ScoringStrategyParams
+	LeaderElection  *LeaderElectionParams
 }
 
 func DecodeSchedulerProfilesFromData(data []byte) ([]ConfigParams, error) {

diff --git a/pkg/objectupdate/sched/render.go b/pkg/objectupdate/sched/render.go
@@ -62,6 +62,29 @@ func RenderConfig(data []byte, schedulerName string, params *manifests.ConfigPar
 
 	updated := false
 
+	if params.LeaderElection != nil {
+		lead, ok, err := unstructured.NestedMap(r.Object, "leaderElection")
+		if !ok || err != nil {
+			klog.ErrorS(err, "failed to process unstructured data", "leaderElection", ok)
+			return data, false, err
+		}
+
+		leadUpdated, err := updateLeaderElection(lead, params)
+		if err != nil {
+			klog.ErrorS(err, "failed to update unstructured data", "leaderElection", lead, "params", params)
+			return data, false, err
+		}
+		if leadUpdated {
+			updated = true
+		}
+
+		if err := unstructured.SetNestedMap(r.Object, lead, "leaderElection"); err != nil {
+			klog.ErrorS(err, "failed to override unstructured data", "data", "leaderElection")
+			return data, false, err
+		}
+
+	}
+
 	profiles, ok, err := unstructured.NestedSlice(r.Object, "profiles")
 	if !ok || err != nil {
 		klog.ErrorS(err, "failed to process unstructured data", "profiles", ok)
@@ -153,6 +176,32 @@ func RenderConfig(data []byte, schedulerName string, params *manifests.ConfigPar
 	return newData, updated, nil
 }
 
+func updateLeaderElection(lead map[string]interface{}, params *manifests.ConfigParams) (bool, error) {
+	var updated int
+	var err error
+
+	err = unstructured.SetNestedField(lead, params.LeaderElection.LeaderElect, "leaderElect")
+	if err != nil {
+		return updated > 0, err
+	}
+	updated++
+
+	err = unstructured.SetNestedField(lead, params.LeaderElection.ResourceName, "resourceName")
+	if err != nil {
+		return updated > 0, err
+	}
+	updated++
+
+	err = unstructured.SetNestedField(lead, params.LeaderElection.ResourceNamespace, "resourceNamespace")
+	if err != nil {
+		return updated > 0, err
+	}
+	updated++
+
+	return updated > 0, nil
+
+}
+
 func updateArgs(args map[string]interface{}, params *manifests.ConfigParams) (bool, error) {
 	var updated int
 	var err error

diff --git a/pkg/options/options.go b/pkg/options/options.go
@@ -40,6 +40,7 @@ type Options struct {
 	SchedResyncPeriod           time.Duration
 	SchedVerbose                int
 	SchedCtrlPlaneAffinity      bool
+	SchedLeaderElection         bool
 	WaitInterval                time.Duration
 	WaitTimeout                 time.Duration
 	ClusterPlatform             platform.Platform
@@ -61,6 +62,7 @@ type Scheduler struct {
 	PullIfNotPresent       bool
 	CacheResyncPeriod      time.Duration
 	CtrlPlaneAffinity      bool
+	LeaderElection         bool
 	Verbose                int
 	ScoringStratConfigData string
 	CacheParamsConfigData  string