From e09e56b4d60c7150c8932f93b0ca111722bf8c7e Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Wed, 20 Mar 2024 08:24:19 +0100 Subject: [PATCH] WIP: sched: support for HA mode WIP TBD Signed-off-by: Francesco Romani --- pkg/commands/deploy.go | 2 ++ pkg/commands/remove.go | 4 ++++ pkg/commands/render.go | 4 ++++ pkg/commands/root.go | 3 ++- pkg/deploy/cluster.go | 2 ++ pkg/deployer/sched/sched.go | 2 +- pkg/manifests/sched/sched.go | 21 +++++++++++++++++++++ pkg/options/options.go | 4 ++++ 8 files changed, 40 insertions(+), 2 deletions(-) diff --git a/pkg/commands/deploy.go b/pkg/commands/deploy.go index d97d35ac..89ccac69 100644 --- a/pkg/commands/deploy.go +++ b/pkg/commands/deploy.go @@ -112,6 +112,8 @@ func NewDeploySchedulerPluginCommand(env *deployer.Environment, commonOpts *opti Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }) }, Args: cobra.NoArgs, diff --git a/pkg/commands/remove.go b/pkg/commands/remove.go index b1c3a593..5181dd38 100644 --- a/pkg/commands/remove.go +++ b/pkg/commands/remove.go @@ -64,6 +64,8 @@ func NewRemoveCommand(env *deployer.Environment, commonOpts *options.Options) *c Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }) if err != nil { // intentionally keep going to remove as much as possible @@ -166,6 +168,8 @@ func NewRemoveSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }) }, Args: cobra.NoArgs, diff --git a/pkg/commands/render.go b/pkg/commands/render.go index 34f60b05..e7e1d150 100644 --- a/pkg/commands/render.go +++ b/pkg/commands/render.go @@ -101,6 +101,8 @@ func NewRenderSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti CacheResyncPeriod: commonOpts.SchedResyncPeriod, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, } schedObjs, err := schedManifests.Render(env.Log, renderOpts) if err != nil { @@ -186,6 +188,8 @@ func RenderManifests(env *deployer.Environment, commonOpts *options.Options) err Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, } schedObjs, err := schedManifests.Render(env.Log, schedRenderOpts) diff --git a/pkg/commands/root.go b/pkg/commands/root.go index 3a28e06e..b6ea4d6e 100644 --- a/pkg/commands/root.go +++ b/pkg/commands/root.go @@ -111,7 +111,8 @@ func InitFlags(flags *pflag.FlagSet, commonOpts *options.Options, internalOpts * flags.StringVar(&commonOpts.SchedProfileName, "sched-profile-name", schedmanifests.DefaultProfileName, "inject scheduler profile name.") flags.DurationVar(&commonOpts.SchedResyncPeriod, "sched-resync-period", schedmanifests.DefaultResyncPeriod, "inject scheduler resync period.") flags.IntVar(&commonOpts.SchedVerbose, "sched-verbose", schedmanifests.DefaultVerbose, "set the scheduler verbosiness.") - flags.BoolVar(&commonOpts.SchedCtrlPlaneAffinity, "sched-ctrlplane-affinity", true, "toggle the scheduler control plane affinity.") + flags.BoolVar(&commonOpts.SchedCtrlPlaneAffinity, "sched-ctrlplane-affinity", schedmanifests.DefaultCtrlPlaneAffinity, "toggle the scheduler control plane affinity.") + flags.StringVar(&commonOpts.SchedLeaderElectResource, "sched-leader-elect-resource", schedmanifests.DefaultLeaderElectResource, "leader election resource namespaced name \"namespace/name\"") } func PostSetupOptions(env *deployer.Environment, commonOpts *options.Options, internalOpts *internalOptions) error { diff --git a/pkg/deploy/cluster.go b/pkg/deploy/cluster.go index 432f853b..2ae58fe0 100644 --- a/pkg/deploy/cluster.go +++ b/pkg/deploy/cluster.go @@ -71,6 +71,8 @@ func OnCluster(env *deployer.Environment, commonOpts *options.Options) error { Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }); err != nil { return err } diff --git a/pkg/deployer/sched/sched.go b/pkg/deployer/sched/sched.go index ab7fa675..f7518347 100644 --- a/pkg/deployer/sched/sched.go +++ b/pkg/deployer/sched/sched.go @@ -37,7 +37,7 @@ func Deploy(env *deployer.Environment, opts options.Scheduler) error { env = env.WithName("SCD") env.Log.Info("deploying topology-aware-scheduling scheduler plugin") - mf, err := schedmanifests.GetManifests(opts.Platform, "") + mf, err := schedmanifests.GetManifests(opts.Platform, "tas-scheduler") if err != nil { return err } diff --git a/pkg/manifests/sched/sched.go b/pkg/manifests/sched/sched.go index 73fe1e4e..5ba54daa 100644 --- a/pkg/manifests/sched/sched.go +++ b/pkg/manifests/sched/sched.go @@ -255,6 +255,27 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) { return mf, nil } +func leaderElectionParamsFromOpts(opts options.Scheduler) (*manifests.LeaderElectionParams, error) { + if !opts.LeaderElection { + return nil, nil + } + leap := manifests.LeaderElectionParams{ + LeaderElect: true, + } + manifests.SetDefaultsLeaderElection(&leap) + var err error + tokens := strings.Split(opts.LeaderElectionResource, "/") + if len(tokens) == 1 { + leap.Namespace = tokens[0] + } else if len(tokens) == 2 { + leap.Namespace = tokens[0] + leap.Name = tokens[1] + } else { + err = fmt.Errorf("malformed leader election resource: %q", opts.LeaderElectionResource) + } + return &leap, err +} + func newInt32(value int32) *int32 { return &value } diff --git a/pkg/options/options.go b/pkg/options/options.go index bb89661f..556a6237 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -40,6 +40,7 @@ type Options struct { SchedResyncPeriod time.Duration SchedVerbose int SchedCtrlPlaneAffinity bool + SchedLeaderElectResource string WaitInterval time.Duration WaitTimeout time.Duration ClusterPlatform platform.Platform @@ -55,12 +56,15 @@ type API struct { type Scheduler struct { Platform platform.Platform + Namespace string WaitCompletion bool Replicas int32 ProfileName string PullIfNotPresent bool CacheResyncPeriod time.Duration CtrlPlaneAffinity bool + LeaderElection bool + LeaderElectionResource string Verbose int ScoringStratConfigData string CacheParamsConfigData string