From 641e6288a9497e137f31298069de709a2d354133 Mon Sep 17 00:00:00 2001 From: Francesco Romani Date: Mon, 4 Mar 2024 15:07:09 +0100 Subject: [PATCH] WIP: sched: support for HA mode WIP TBD Signed-off-by: Francesco Romani --- pkg/commands/deploy.go | 2 + pkg/commands/remove.go | 4 + pkg/commands/render.go | 4 + pkg/commands/root.go | 3 +- pkg/deploy/cluster.go | 2 + pkg/deployer/sched/sched.go | 2 +- pkg/manifests/manifests.go | 8 +- pkg/manifests/manifests_test.go | 6 +- pkg/manifests/rte/rte.go | 2 +- pkg/manifests/sched/sched.go | 98 +++++++++++++------ pkg/manifests/schedparams.go | 21 ++++ .../yaml/sched/scheduler/clusterrole.yaml | 14 --- pkg/manifests/yaml/sched/scheduler/role.yaml | 19 ++++ ...binding.yaml => rolebinding_authread.yaml} | 0 .../scheduler/rolebinding_leaderelect.yaml | 13 +++ pkg/objectupdate/sched/render.go | 49 ++++++++++ pkg/objectwait/sched/sched.go | 8 +- pkg/options/options.go | 4 + 18 files changed, 207 insertions(+), 52 deletions(-) create mode 100644 pkg/manifests/yaml/sched/scheduler/role.yaml rename pkg/manifests/yaml/sched/scheduler/{rolebinding.yaml => rolebinding_authread.yaml} (100%) create mode 100644 pkg/manifests/yaml/sched/scheduler/rolebinding_leaderelect.yaml diff --git a/pkg/commands/deploy.go b/pkg/commands/deploy.go index d97d35ac..89ccac69 100644 --- a/pkg/commands/deploy.go +++ b/pkg/commands/deploy.go @@ -112,6 +112,8 @@ func NewDeploySchedulerPluginCommand(env *deployer.Environment, commonOpts *opti Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }) }, Args: cobra.NoArgs, diff --git a/pkg/commands/remove.go b/pkg/commands/remove.go index b1c3a593..5181dd38 100644 --- a/pkg/commands/remove.go +++ b/pkg/commands/remove.go @@ -64,6 +64,8 @@ func NewRemoveCommand(env *deployer.Environment, commonOpts *options.Options) *c Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }) if err != nil { // intentionally keep going to remove as much as possible @@ -166,6 +168,8 @@ func NewRemoveSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }) }, Args: cobra.NoArgs, diff --git a/pkg/commands/render.go b/pkg/commands/render.go index 34f60b05..e7e1d150 100644 --- a/pkg/commands/render.go +++ b/pkg/commands/render.go @@ -101,6 +101,8 @@ func NewRenderSchedulerPluginCommand(env *deployer.Environment, commonOpts *opti CacheResyncPeriod: commonOpts.SchedResyncPeriod, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, } schedObjs, err := schedManifests.Render(env.Log, renderOpts) if err != nil { @@ -186,6 +188,8 @@ func RenderManifests(env *deployer.Environment, commonOpts *options.Options) err Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, } schedObjs, err := schedManifests.Render(env.Log, schedRenderOpts) diff --git a/pkg/commands/root.go b/pkg/commands/root.go index 3a28e06e..b6ea4d6e 100644 --- a/pkg/commands/root.go +++ b/pkg/commands/root.go @@ -111,7 +111,8 @@ func InitFlags(flags *pflag.FlagSet, commonOpts *options.Options, internalOpts * flags.StringVar(&commonOpts.SchedProfileName, "sched-profile-name", schedmanifests.DefaultProfileName, "inject scheduler profile name.") flags.DurationVar(&commonOpts.SchedResyncPeriod, "sched-resync-period", schedmanifests.DefaultResyncPeriod, "inject scheduler resync period.") flags.IntVar(&commonOpts.SchedVerbose, "sched-verbose", schedmanifests.DefaultVerbose, "set the scheduler verbosiness.") - flags.BoolVar(&commonOpts.SchedCtrlPlaneAffinity, "sched-ctrlplane-affinity", true, "toggle the scheduler control plane affinity.") + flags.BoolVar(&commonOpts.SchedCtrlPlaneAffinity, "sched-ctrlplane-affinity", schedmanifests.DefaultCtrlPlaneAffinity, "toggle the scheduler control plane affinity.") + flags.StringVar(&commonOpts.SchedLeaderElectResource, "sched-leader-elect-resource", schedmanifests.DefaultLeaderElectResource, "leader election resource namespaced name \"namespace/name\"") } func PostSetupOptions(env *deployer.Environment, commonOpts *options.Options, internalOpts *internalOptions) error { diff --git a/pkg/deploy/cluster.go b/pkg/deploy/cluster.go index 432f853b..2ae58fe0 100644 --- a/pkg/deploy/cluster.go +++ b/pkg/deploy/cluster.go @@ -71,6 +71,8 @@ func OnCluster(env *deployer.Environment, commonOpts *options.Options) error { Verbose: commonOpts.SchedVerbose, ScoringStratConfigData: commonOpts.SchedScoringStratConfigData, CacheParamsConfigData: commonOpts.SchedCacheParamsConfigData, + LeaderElection: commonOpts.Replicas > 1, + LeaderElectionResource: commonOpts.SchedLeaderElectResource, }); err != nil { return err } diff --git a/pkg/deployer/sched/sched.go b/pkg/deployer/sched/sched.go index ab7fa675..f7518347 100644 --- a/pkg/deployer/sched/sched.go +++ b/pkg/deployer/sched/sched.go @@ -37,7 +37,7 @@ func Deploy(env *deployer.Environment, opts options.Scheduler) error { env = env.WithName("SCD") env.Log.Info("deploying topology-aware-scheduling scheduler plugin") - mf, err := schedmanifests.GetManifests(opts.Platform, "") + mf, err := schedmanifests.GetManifests(opts.Platform, "tas-scheduler") if err != nil { return err } diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go index 25216c3f..bf198d13 100644 --- a/pkg/manifests/manifests.go +++ b/pkg/manifests/manifests.go @@ -148,7 +148,7 @@ func Role(component, subComponent, namespace string) (*rbacv1.Role, error) { return role, nil } -func RoleBinding(component, subComponent, namespace string) (*rbacv1.RoleBinding, error) { +func RoleBinding(component, subComponent, roleName, namespace string) (*rbacv1.RoleBinding, error) { if err := validateComponent(component); err != nil { return nil, err } @@ -156,7 +156,11 @@ func RoleBinding(component, subComponent, namespace string) (*rbacv1.RoleBinding return nil, err } - obj, err := loadObject(filepath.Join("yaml", component, subComponent, "rolebinding.yaml")) + fileName := "rolebinding.yaml" + if roleName != "" { + fileName = "rolebinding_" + roleName + ".yaml" + } + obj, err := loadObject(filepath.Join("yaml", component, subComponent, fileName)) if err != nil { return nil, err } diff --git a/pkg/manifests/manifests_test.go b/pkg/manifests/manifests_test.go index a1f33a00..abb1087f 100644 --- a/pkg/manifests/manifests_test.go +++ b/pkg/manifests/manifests_test.go @@ -126,7 +126,7 @@ func TestGetRole(t *testing.T) { { component: ComponentSchedulerPlugin, subComponent: SubComponentSchedulerPluginScheduler, - expectError: true, + expectError: false, }, { component: ComponentSchedulerPlugin, @@ -159,6 +159,7 @@ func TestGetRoleBinding(t *testing.T) { type testCase struct { component string subComponent string + roleName string expectError bool } @@ -174,6 +175,7 @@ func TestGetRoleBinding(t *testing.T) { { component: ComponentSchedulerPlugin, subComponent: SubComponentSchedulerPluginScheduler, + roleName: "authread", expectError: false, }, { @@ -189,7 +191,7 @@ func TestGetRoleBinding(t *testing.T) { for _, tc := range testCases { t.Run(tc.component, func(t *testing.T) { - obj, err := RoleBinding(tc.component, tc.subComponent, "") + obj, err := RoleBinding(tc.component, tc.subComponent, tc.roleName, "") if tc.expectError { if err == nil || obj != nil { t.Fatalf("nil err or non-nil obj=%v", obj) diff --git a/pkg/manifests/rte/rte.go b/pkg/manifests/rte/rte.go index d4f1d13d..7cdaf265 100644 --- a/pkg/manifests/rte/rte.go +++ b/pkg/manifests/rte/rte.go @@ -197,7 +197,7 @@ func GetManifests(plat platform.Platform, version platform.Version, namespace st if err != nil { return mf, err } - mf.RoleBinding, err = manifests.RoleBinding(manifests.ComponentResourceTopologyExporter, "", namespace) + mf.RoleBinding, err = manifests.RoleBinding(manifests.ComponentResourceTopologyExporter, "", "", namespace) if err != nil { return mf, err } diff --git a/pkg/manifests/sched/sched.go b/pkg/manifests/sched/sched.go index c4e06345..cbd252e2 100644 --- a/pkg/manifests/sched/sched.go +++ b/pkg/manifests/sched/sched.go @@ -19,6 +19,7 @@ package sched import ( "encoding/json" "fmt" + "strings" "time" "github.com/go-logr/logr" @@ -39,9 +40,11 @@ import ( ) const ( - DefaultProfileName = "topology-aware-scheduler" - DefaultResyncPeriod = 5 * time.Second - DefaultVerbose = 4 + DefaultProfileName = "topology-aware-scheduler" + DefaultResyncPeriod = 5 * time.Second + DefaultVerbose = 4 + DefaultCtrlPlaneAffinity = true + DefaultLeaderElectResource = manifests.LeaderElectionDefaultNamespace + "/" + manifests.LeaderElectionDefaultName ) const ( @@ -59,12 +62,14 @@ type Manifests struct { RBController *rbacv1.RoleBinding DPController *appsv1.Deployment // scheduler proper - SAScheduler *corev1.ServiceAccount - CRScheduler *rbacv1.ClusterRole - CRBScheduler *rbacv1.ClusterRoleBinding - RBScheduler *rbacv1.RoleBinding - DPScheduler *appsv1.Deployment - ConfigMap *corev1.ConfigMap + SAScheduler *corev1.ServiceAccount + CRScheduler *rbacv1.ClusterRole + RSchedulerElect *rbacv1.Role + CRBScheduler *rbacv1.ClusterRoleBinding + RBSchedulerAuth *rbacv1.RoleBinding + RBSchedulerElect *rbacv1.RoleBinding + DPScheduler *appsv1.Deployment + ConfigMap *corev1.ConfigMap // internal fields plat platform.Platform } @@ -73,19 +78,21 @@ func (mf Manifests) Clone() Manifests { return Manifests{ plat: mf.plat, // objects - Crd: mf.Crd.DeepCopy(), - Namespace: mf.Namespace.DeepCopy(), - SAController: mf.SAController.DeepCopy(), - CRController: mf.CRController.DeepCopy(), - CRBController: mf.CRBController.DeepCopy(), - DPController: mf.DPController.DeepCopy(), - RBController: mf.RBController.DeepCopy(), - SAScheduler: mf.SAScheduler.DeepCopy(), - CRScheduler: mf.CRScheduler.DeepCopy(), - CRBScheduler: mf.CRBScheduler.DeepCopy(), - DPScheduler: mf.DPScheduler.DeepCopy(), - ConfigMap: mf.ConfigMap.DeepCopy(), - RBScheduler: mf.RBScheduler.DeepCopy(), + Crd: mf.Crd.DeepCopy(), + Namespace: mf.Namespace.DeepCopy(), + SAController: mf.SAController.DeepCopy(), + CRController: mf.CRController.DeepCopy(), + CRBController: mf.CRBController.DeepCopy(), + DPController: mf.DPController.DeepCopy(), + RBController: mf.RBController.DeepCopy(), + SAScheduler: mf.SAScheduler.DeepCopy(), + CRScheduler: mf.CRScheduler.DeepCopy(), + RSchedulerElect: mf.RSchedulerElect.DeepCopy(), + CRBScheduler: mf.CRBScheduler.DeepCopy(), + RBSchedulerAuth: mf.RBSchedulerAuth.DeepCopy(), + RBSchedulerElect: mf.RBSchedulerElect.DeepCopy(), + DPScheduler: mf.DPScheduler.DeepCopy(), + ConfigMap: mf.ConfigMap.DeepCopy(), } } @@ -104,6 +111,11 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest Cache: manifests.NewConfigCacheParams(), } + params.LeaderElection, err = leaderElectionParamsFromOpts(opts) + if err != nil { + return ret, err + } + if len(opts.CacheParamsConfigData) > 0 { err = yaml.Unmarshal([]byte(opts.CacheParamsConfigData), params.Cache) if err != nil { @@ -111,9 +123,6 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest } } - // always override - params.Cache.ResyncPeriodSeconds = newInt64(int64(opts.CacheResyncPeriod.Seconds())) - if len(opts.ScoringStratConfigData) > 0 { params.ScoringStrategy = &manifests.ScoringStrategyParams{} err = yaml.Unmarshal([]byte(opts.ScoringStratConfigData), params.ScoringStrategy) @@ -122,6 +131,9 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest } } + // always override + params.Cache.ResyncPeriodSeconds = newInt64(int64(opts.CacheResyncPeriod.Seconds())) + err = schedupdate.SchedulerConfig(ret.ConfigMap, DefaultProfileName, ¶ms) if err != nil { return ret, err @@ -140,7 +152,8 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest ret.SAScheduler.Namespace = ret.Namespace.Name rbacupdate.ClusterRoleBinding(ret.CRBScheduler, ret.SAScheduler.Name, ret.Namespace.Name) - rbacupdate.RoleBinding(ret.RBScheduler, ret.SAScheduler.Name, ret.Namespace.Name) + rbacupdate.RoleBinding(ret.RBSchedulerElect, ret.SAScheduler.Name, ret.Namespace.Name) + rbacupdate.RoleBinding(ret.RBSchedulerAuth, ret.SAScheduler.Name, ret.Namespace.Name) ret.DPScheduler.Namespace = ret.Namespace.Name ret.ConfigMap.Namespace = ret.Namespace.Name @@ -155,7 +168,9 @@ func (mf Manifests) ToObjects() []client.Object { mf.CRScheduler, mf.CRBScheduler, mf.ConfigMap, - mf.RBScheduler, + mf.RSchedulerElect, + mf.RBSchedulerAuth, + mf.RBSchedulerElect, mf.DPScheduler, mf.SAController, mf.CRController, @@ -199,7 +214,15 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) { if err != nil { return mf, err } - mf.RBScheduler, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, namespace) + mf.RSchedulerElect, err = manifests.Role(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, namespace) + if err != nil { + return mf, err + } + mf.RBSchedulerElect, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, "leaderelect", namespace) + if err != nil { + return mf, err + } + mf.RBSchedulerAuth, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, "authread", namespace) if err != nil { return mf, err } @@ -220,7 +243,7 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) { if err != nil { return mf, err } - mf.RBController, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginController, namespace) + mf.RBController, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginController, "", namespace) if err != nil { return mf, err } @@ -232,6 +255,23 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) { return mf, nil } +func leaderElectionParamsFromOpts(opts options.Scheduler) (*manifests.LeaderElectionParams, error) { + if !opts.LeaderElection { + return nil, nil + } + tokens := strings.Split(opts.LeaderElectionResource, "/") + if len(tokens) != 2 { + return nil, fmt.Errorf("malformed leader election resource: %q", opts.LeaderElectionResource) + } + leap := manifests.LeaderElectionParams{ + LeaderElect: true, + ResourceNamespace: tokens[0], + ResourceName: tokens[1], + } + manifests.SetDefaultsLeaderElection(&leap) + return &leap, nil +} + func newInt32(value int32) *int32 { return &value } diff --git a/pkg/manifests/schedparams.go b/pkg/manifests/schedparams.go index 3c5c7b78..c8fb0c26 100644 --- a/pkg/manifests/schedparams.go +++ b/pkg/manifests/schedparams.go @@ -54,6 +54,11 @@ const ( ScoringStrategyLeastAllocated = "LeastAllocated" ) +const ( + LeaderElectionDefaultName = "nrtmatch-scheduler" + LeaderElectionDefaultNamespace = "tas-scheduler" +) + func ValidateForeignPodsDetectMode(value string) error { switch value { case ForeignPodsDetectNone: @@ -136,10 +141,26 @@ func ValidateScoringStrategyType(value string) error { } } +type LeaderElectionParams struct { + LeaderElect bool `json:"leaderElect,omitempty"` + ResourceName string `json:"resourceName,omitempty"` + ResourceNamespace string `json:"resourceNamespace,omitempty"` +} + +func SetDefaultsLeaderElection(lep *LeaderElectionParams) { + if lep.ResourceName == "" { + lep.ResourceName = LeaderElectionDefaultName + } + if lep.ResourceNamespace == "" { + lep.ResourceNamespace = LeaderElectionDefaultNamespace + } +} + type ConfigParams struct { ProfileName string // can't be empty, so no need for pointer Cache *ConfigCacheParams ScoringStrategy *ScoringStrategyParams + LeaderElection *LeaderElectionParams } func DecodeSchedulerProfilesFromData(data []byte) ([]ConfigParams, error) { diff --git a/pkg/manifests/yaml/sched/scheduler/clusterrole.yaml b/pkg/manifests/yaml/sched/scheduler/clusterrole.yaml index 0def0f9d..0de0a64f 100644 --- a/pkg/manifests/yaml/sched/scheduler/clusterrole.yaml +++ b/pkg/manifests/yaml/sched/scheduler/clusterrole.yaml @@ -6,20 +6,6 @@ rules: - apiGroups: ["", "events.k8s.io"] resources: ["events"] verbs: ["create", "patch", "update"] -- apiGroups: ["coordination.k8s.io"] - resources: ["leases"] - verbs: ["create"] -- apiGroups: ["coordination.k8s.io"] - resourceNames: ["kube-scheduler"] - resources: ["leases"] - verbs: ["get", "update"] -- apiGroups: [""] - resources: ["endpoints"] - verbs: ["create"] -- apiGroups: [""] - resourceNames: ["kube-scheduler"] - resources: ["endpoints"] - verbs: ["get", "update"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "watch"] diff --git a/pkg/manifests/yaml/sched/scheduler/role.yaml b/pkg/manifests/yaml/sched/scheduler/role.yaml new file mode 100644 index 00000000..cc68827a --- /dev/null +++ b/pkg/manifests/yaml/sched/scheduler/role.yaml @@ -0,0 +1,19 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: topology-aware-scheduler-leader-elect +rules: +- apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["create"] +- apiGroups: ["coordination.k8s.io"] + resourceNames: ["nrtmatch-scheduler"] + resources: ["leases"] + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["create"] +- apiGroups: [""] + resourceNames: ["nrtmatch-scheduler"] + resources: ["endpoints"] + verbs: ["get", "update"] diff --git a/pkg/manifests/yaml/sched/scheduler/rolebinding.yaml b/pkg/manifests/yaml/sched/scheduler/rolebinding_authread.yaml similarity index 100% rename from pkg/manifests/yaml/sched/scheduler/rolebinding.yaml rename to pkg/manifests/yaml/sched/scheduler/rolebinding_authread.yaml diff --git a/pkg/manifests/yaml/sched/scheduler/rolebinding_leaderelect.yaml b/pkg/manifests/yaml/sched/scheduler/rolebinding_leaderelect.yaml new file mode 100644 index 00000000..cdbe3a5e --- /dev/null +++ b/pkg/manifests/yaml/sched/scheduler/rolebinding_leaderelect.yaml @@ -0,0 +1,13 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: topology-aware-scheduler-leader-elect + namespace: tas-scheduler +subjects: + - kind: ServiceAccount + name: topology-aware-scheduler + namespace: tas-scheduler +roleRef: + kind: Role + name: topology-aware-scheduler-leader-elect + apiGroup: rbac.authorization.k8s.io diff --git a/pkg/objectupdate/sched/render.go b/pkg/objectupdate/sched/render.go index a7d55961..6ada0af0 100644 --- a/pkg/objectupdate/sched/render.go +++ b/pkg/objectupdate/sched/render.go @@ -62,6 +62,29 @@ func RenderConfig(data []byte, schedulerName string, params *manifests.ConfigPar updated := false + if params.LeaderElection != nil { + lead, ok, err := unstructured.NestedMap(r.Object, "leaderElection") + if !ok || err != nil { + klog.ErrorS(err, "failed to process unstructured data", "leaderElection", ok) + return data, false, err + } + + leadUpdated, err := updateLeaderElection(lead, params) + if err != nil { + klog.ErrorS(err, "failed to update unstructured data", "leaderElection", lead, "params", params) + return data, false, err + } + if leadUpdated { + updated = true + } + + if err := unstructured.SetNestedMap(r.Object, lead, "leaderElection"); err != nil { + klog.ErrorS(err, "failed to override unstructured data", "data", "leaderElection") + return data, false, err + } + + } + profiles, ok, err := unstructured.NestedSlice(r.Object, "profiles") if !ok || err != nil { klog.ErrorS(err, "failed to process unstructured data", "profiles", ok) @@ -153,6 +176,32 @@ func RenderConfig(data []byte, schedulerName string, params *manifests.ConfigPar return newData, updated, nil } +func updateLeaderElection(lead map[string]interface{}, params *manifests.ConfigParams) (bool, error) { + var updated int + var err error + + err = unstructured.SetNestedField(lead, params.LeaderElection.LeaderElect, "leaderElect") + if err != nil { + return updated > 0, err + } + updated++ + + err = unstructured.SetNestedField(lead, params.LeaderElection.ResourceName, "resourceName") + if err != nil { + return updated > 0, err + } + updated++ + + err = unstructured.SetNestedField(lead, params.LeaderElection.ResourceNamespace, "resourceNamespace") + if err != nil { + return updated > 0, err + } + updated++ + + return updated > 0, nil + +} + func updateArgs(args map[string]interface{}, params *manifests.ConfigParams) (bool, error) { var updated int var err error diff --git a/pkg/objectwait/sched/sched.go b/pkg/objectwait/sched/sched.go index 4baf8ae8..b342b1c1 100644 --- a/pkg/objectwait/sched/sched.go +++ b/pkg/objectwait/sched/sched.go @@ -35,7 +35,9 @@ func Creatable(mf schedmf.Manifests, cli client.Client, log logr.Logger) []objec {Obj: mf.SAScheduler}, {Obj: mf.CRScheduler}, {Obj: mf.CRBScheduler}, - {Obj: mf.RBScheduler}, + {Obj: mf.RSchedulerElect}, + {Obj: mf.RBSchedulerElect}, + {Obj: mf.RBSchedulerAuth}, {Obj: mf.ConfigMap}, { Obj: mf.DPScheduler, @@ -69,7 +71,9 @@ func Deletable(mf schedmf.Manifests, cli client.Client, log logr.Logger) []objec // no need to remove objects created inside the namespace we just removed {Obj: mf.CRBScheduler}, {Obj: mf.CRScheduler}, - {Obj: mf.RBScheduler}, + {Obj: mf.RBSchedulerAuth}, + {Obj: mf.RBSchedulerElect}, + {Obj: mf.RSchedulerElect}, {Obj: mf.CRBController}, {Obj: mf.CRController}, {Obj: mf.RBController}, diff --git a/pkg/options/options.go b/pkg/options/options.go index bb89661f..556a6237 100644 --- a/pkg/options/options.go +++ b/pkg/options/options.go @@ -40,6 +40,7 @@ type Options struct { SchedResyncPeriod time.Duration SchedVerbose int SchedCtrlPlaneAffinity bool + SchedLeaderElectResource string WaitInterval time.Duration WaitTimeout time.Duration ClusterPlatform platform.Platform @@ -55,12 +56,15 @@ type API struct { type Scheduler struct { Platform platform.Platform + Namespace string WaitCompletion bool Replicas int32 ProfileName string PullIfNotPresent bool CacheResyncPeriod time.Duration CtrlPlaneAffinity bool + LeaderElection bool + LeaderElectionResource string Verbose int ScoringStratConfigData string CacheParamsConfigData string