WIP: sched: support for HA mode

WIP TBD Signed-off-by: Francesco Romani <[email protected]>
k8stopologyawareschedwg · Mar 20, 2024 · e09e56b · e09e56b
1 parent a1bd4cb
commit e09e56b
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 2 deletions.
diff --git a/pkg/commands/deploy.go b/pkg/commands/deploy.go
@@ -112,6 +112,8 @@ func NewDeploySchedulerPluginCommand(env *deployer.Environment, commonOpts *opti
 				Verbose:                commonOpts.SchedVerbose,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
+				LeaderElectionResource: commonOpts.SchedLeaderElectResource,
 			})
 		},
 		Args: cobra.NoArgs,

diff --git a/pkg/commands/remove.go b/pkg/commands/remove.go
@@ -64,6 +64,8 @@ func NewRemoveCommand(env *deployer.Environment, commonOpts *options.Options) *c
 				Verbose:                commonOpts.SchedVerbose,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
+				LeaderElectionResource: commonOpts.SchedLeaderElectResource,
 			})
 			if err != nil {
 				// intentionally keep going to remove as much as possible
@@ -166,6 +168,8 @@ func NewRemoveSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti
 				Verbose:                commonOpts.SchedVerbose,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
+				LeaderElectionResource: commonOpts.SchedLeaderElectResource,
 			})
 		},
 		Args: cobra.NoArgs,

diff --git a/pkg/commands/render.go b/pkg/commands/render.go
@@ -101,6 +101,8 @@ func NewRenderSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti
 				CacheResyncPeriod:      commonOpts.SchedResyncPeriod,
 				ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 				CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+				LeaderElection:         commonOpts.Replicas > 1,
+				LeaderElectionResource: commonOpts.SchedLeaderElectResource,
 			}
 			schedObjs, err := schedManifests.Render(env.Log, renderOpts)
 			if err != nil {
@@ -186,6 +188,8 @@ func RenderManifests(env *deployer.Environment, commonOpts *options.Options) err
 		Verbose:                commonOpts.SchedVerbose,
 		ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 		CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+		LeaderElection:         commonOpts.Replicas > 1,
+		LeaderElectionResource: commonOpts.SchedLeaderElectResource,
 	}
 
 	schedObjs, err := schedManifests.Render(env.Log, schedRenderOpts)

diff --git a/pkg/commands/root.go b/pkg/commands/root.go
@@ -111,7 +111,8 @@ func InitFlags(flags *pflag.FlagSet, commonOpts *options.Options, internalOpts *
 	flags.StringVar(&commonOpts.SchedProfileName, "sched-profile-name", schedmanifests.DefaultProfileName, "inject scheduler profile name.")
 	flags.DurationVar(&commonOpts.SchedResyncPeriod, "sched-resync-period", schedmanifests.DefaultResyncPeriod, "inject scheduler resync period.")
 	flags.IntVar(&commonOpts.SchedVerbose, "sched-verbose", schedmanifests.DefaultVerbose, "set the scheduler verbosiness.")
-	flags.BoolVar(&commonOpts.SchedCtrlPlaneAffinity, "sched-ctrlplane-affinity", true, "toggle the scheduler control plane affinity.")
+	flags.BoolVar(&commonOpts.SchedCtrlPlaneAffinity, "sched-ctrlplane-affinity", schedmanifests.DefaultCtrlPlaneAffinity, "toggle the scheduler control plane affinity.")
+	flags.StringVar(&commonOpts.SchedLeaderElectResource, "sched-leader-elect-resource", schedmanifests.DefaultLeaderElectResource, "leader election resource namespaced name \"namespace/name\"")
 }
 
 func PostSetupOptions(env *deployer.Environment, commonOpts *options.Options, internalOpts *internalOptions) error {

diff --git a/pkg/deploy/cluster.go b/pkg/deploy/cluster.go
@@ -71,6 +71,8 @@ func OnCluster(env *deployer.Environment, commonOpts *options.Options) error {
 		Verbose:                commonOpts.SchedVerbose,
 		ScoringStratConfigData: commonOpts.SchedScoringStratConfigData,
 		CacheParamsConfigData:  commonOpts.SchedCacheParamsConfigData,
+		LeaderElection:         commonOpts.Replicas > 1,
+		LeaderElectionResource: commonOpts.SchedLeaderElectResource,
 	}); err != nil {
 		return err
 	}

diff --git a/pkg/deployer/sched/sched.go b/pkg/deployer/sched/sched.go
@@ -37,7 +37,7 @@ func Deploy(env *deployer.Environment, opts options.Scheduler) error {
 	env = env.WithName("SCD")
 	env.Log.Info("deploying topology-aware-scheduling scheduler plugin")
 
-	mf, err := schedmanifests.GetManifests(opts.Platform, "")
+	mf, err := schedmanifests.GetManifests(opts.Platform, "tas-scheduler")
 	if err != nil {
 		return err
 	}

diff --git a/pkg/manifests/sched/sched.go b/pkg/manifests/sched/sched.go
@@ -255,6 +255,27 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) {
 	return mf, nil
 }
 
+func leaderElectionParamsFromOpts(opts options.Scheduler) (*manifests.LeaderElectionParams, error) {
+	if !opts.LeaderElection {
+		return nil, nil
+	}
+	leap := manifests.LeaderElectionParams{
+		LeaderElect: true,
+	}
+	manifests.SetDefaultsLeaderElection(&leap)
+	var err error
+	tokens := strings.Split(opts.LeaderElectionResource, "/")
+	if len(tokens) == 1 {
+		leap.Namespace = tokens[0]
+	} else if len(tokens) == 2 {
+		leap.Namespace = tokens[0]
+		leap.Name = tokens[1]
+	} else {
+		err = fmt.Errorf("malformed leader election resource: %q", opts.LeaderElectionResource)
+	}
+	return &leap, err
+}
+
 func newInt32(value int32) *int32 {
 	return &value
 }

diff --git a/pkg/options/options.go b/pkg/options/options.go
@@ -40,6 +40,7 @@ type Options struct {
 	SchedResyncPeriod           time.Duration
 	SchedVerbose                int
 	SchedCtrlPlaneAffinity      bool
+	SchedLeaderElectResource    string
 	WaitInterval                time.Duration
 	WaitTimeout                 time.Duration
 	ClusterPlatform             platform.Platform
@@ -55,12 +56,15 @@ type API struct {
 
 type Scheduler struct {
 	Platform               platform.Platform
+	Namespace              string
 	WaitCompletion         bool
 	Replicas               int32
 	ProfileName            string
 	PullIfNotPresent       bool
 	CacheResyncPeriod      time.Duration
 	CtrlPlaneAffinity      bool
+	LeaderElection         bool
+	LeaderElectionResource string
 	Verbose                int
 	ScoringStratConfigData string
 	CacheParamsConfigData  string