manifests: sched: role: add RBAC data for leader election

We now need to add a new set of permissions, including
roles (+bindings) to enable leader election.
Add support in the manifests.

Signed-off-by: Francesco Romani <[email protected]>

pkg/manifests/manifests.go

@@ -148,15 +148,19 @@ func Role(component, subComponent, namespace string) (*rbacv1.Role, error) {
 	return role, nil
 }
 
-func RoleBinding(component, subComponent, namespace string) (*rbacv1.RoleBinding, error) {
+func RoleBinding(component, subComponent, roleName, namespace string) (*rbacv1.RoleBinding, error) {
 	if err := validateComponent(component); err != nil {
 		return nil, err
 	}
 	if err := validateSubComponent(component, subComponent); err != nil {
 		return nil, err
 	}
 
-	obj, err := loadObject(filepath.Join("yaml", component, subComponent, "rolebinding.yaml"))
+	fileName := "rolebinding.yaml"
+	if roleName != "" {
+		fileName = "rolebinding_" + roleName + ".yaml"
+	}
+	obj, err := loadObject(filepath.Join("yaml", component, subComponent, fileName))
 	if err != nil {
 		return nil, err
 	}

pkg/manifests/manifests_test.go

@@ -126,7 +126,7 @@ func TestGetRole(t *testing.T) {
 		{
 			component:    ComponentSchedulerPlugin,
 			subComponent: SubComponentSchedulerPluginScheduler,
-			expectError:  true,
+			expectError:  false,
 		},
 		{
 			component:    ComponentSchedulerPlugin,
@@ -159,6 +159,7 @@ func TestGetRoleBinding(t *testing.T) {
 	type testCase struct {
 		component    string
 		subComponent string
+		roleName     string
 		expectError  bool
 	}
 
@@ -174,6 +175,7 @@ func TestGetRoleBinding(t *testing.T) {
 		{
 			component:    ComponentSchedulerPlugin,
 			subComponent: SubComponentSchedulerPluginScheduler,
+			roleName:     "authread",
 			expectError:  false,
 		},
 		{
@@ -189,7 +191,7 @@ func TestGetRoleBinding(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.component, func(t *testing.T) {
-			obj, err := RoleBinding(tc.component, tc.subComponent, "")
+			obj, err := RoleBinding(tc.component, tc.subComponent, tc.roleName, "")
 			if tc.expectError {
 				if err == nil || obj != nil {
 					t.Fatalf("nil err or non-nil obj=%v", obj)

pkg/manifests/rte/rte.go

@@ -197,7 +197,7 @@ func GetManifests(plat platform.Platform, version platform.Version, namespace st
 	if err != nil {
 		return mf, err
 	}
-	mf.RoleBinding, err = manifests.RoleBinding(manifests.ComponentResourceTopologyExporter, "", namespace)
+	mf.RoleBinding, err = manifests.RoleBinding(manifests.ComponentResourceTopologyExporter, "", "", namespace)
 	if err != nil {
 		return mf, err
 	}

pkg/manifests/sched/sched.go

@@ -19,6 +19,7 @@ package sched
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -61,12 +62,14 @@ type Manifests struct {
 	RBController  *rbacv1.RoleBinding
 	DPController  *appsv1.Deployment
 	// scheduler proper
-	SAScheduler  *corev1.ServiceAccount
-	CRScheduler  *rbacv1.ClusterRole
-	CRBScheduler *rbacv1.ClusterRoleBinding
-	RBScheduler  *rbacv1.RoleBinding
-	DPScheduler  *appsv1.Deployment
-	ConfigMap    *corev1.ConfigMap
+	SAScheduler      *corev1.ServiceAccount
+	CRScheduler      *rbacv1.ClusterRole
+	RSchedulerElect  *rbacv1.Role
+	CRBScheduler     *rbacv1.ClusterRoleBinding
+	RBSchedulerAuth  *rbacv1.RoleBinding
+	RBSchedulerElect *rbacv1.RoleBinding
+	DPScheduler      *appsv1.Deployment
+	ConfigMap        *corev1.ConfigMap
 	// internal fields
 	plat platform.Platform
 }
@@ -75,19 +78,21 @@ func (mf Manifests) Clone() Manifests {
 	return Manifests{
 		plat: mf.plat,
 		// objects
-		Crd:           mf.Crd.DeepCopy(),
-		Namespace:     mf.Namespace.DeepCopy(),
-		SAController:  mf.SAController.DeepCopy(),
-		CRController:  mf.CRController.DeepCopy(),
-		CRBController: mf.CRBController.DeepCopy(),
-		DPController:  mf.DPController.DeepCopy(),
-		RBController:  mf.RBController.DeepCopy(),
-		SAScheduler:   mf.SAScheduler.DeepCopy(),
-		CRScheduler:   mf.CRScheduler.DeepCopy(),
-		CRBScheduler:  mf.CRBScheduler.DeepCopy(),
-		DPScheduler:   mf.DPScheduler.DeepCopy(),
-		ConfigMap:     mf.ConfigMap.DeepCopy(),
-		RBScheduler:   mf.RBScheduler.DeepCopy(),
+		Crd:              mf.Crd.DeepCopy(),
+		Namespace:        mf.Namespace.DeepCopy(),
+		SAController:     mf.SAController.DeepCopy(),
+		CRController:     mf.CRController.DeepCopy(),
+		CRBController:    mf.CRBController.DeepCopy(),
+		DPController:     mf.DPController.DeepCopy(),
+		RBController:     mf.RBController.DeepCopy(),
+		SAScheduler:      mf.SAScheduler.DeepCopy(),
+		CRScheduler:      mf.CRScheduler.DeepCopy(),
+		RSchedulerElect:  mf.RSchedulerElect.DeepCopy(),
+		CRBScheduler:     mf.CRBScheduler.DeepCopy(),
+		RBSchedulerAuth:  mf.RBSchedulerAuth.DeepCopy(),
+		RBSchedulerElect: mf.RBSchedulerElect.DeepCopy(),
+		DPScheduler:      mf.DPScheduler.DeepCopy(),
+		ConfigMap:        mf.ConfigMap.DeepCopy(),
 	}
 }
 
@@ -106,16 +111,18 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest
 		Cache:       manifests.NewConfigCacheParams(),
 	}
 
+	params.LeaderElection, err = leaderElectionParamsFromOpts(opts)
+	if err != nil {
+		return ret, err
+	}
+
 	if len(opts.CacheParamsConfigData) > 0 {
 		err = yaml.Unmarshal([]byte(opts.CacheParamsConfigData), params.Cache)
 		if err != nil {
 			return ret, err
 		}
 	}
 
-	// always override
-	params.Cache.ResyncPeriodSeconds = newInt64(int64(opts.CacheResyncPeriod.Seconds()))
-
 	if len(opts.ScoringStratConfigData) > 0 {
 		params.ScoringStrategy = &manifests.ScoringStrategyParams{}
 		err = yaml.Unmarshal([]byte(opts.ScoringStratConfigData), params.ScoringStrategy)
@@ -124,6 +131,9 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest
 		}
 	}
 
+	// always override
+	params.Cache.ResyncPeriodSeconds = newInt64(int64(opts.CacheResyncPeriod.Seconds()))
+
 	err = schedupdate.SchedulerConfig(ret.ConfigMap, DefaultProfileName, &params)
 	if err != nil {
 		return ret, err
@@ -142,7 +152,8 @@ func (mf Manifests) Render(logger logr.Logger, opts options.Scheduler) (Manifest
 
 	ret.SAScheduler.Namespace = ret.Namespace.Name
 	rbacupdate.ClusterRoleBinding(ret.CRBScheduler, ret.SAScheduler.Name, ret.Namespace.Name)
-	rbacupdate.RoleBinding(ret.RBScheduler, ret.SAScheduler.Name, ret.Namespace.Name)
+	rbacupdate.RoleBinding(ret.RBSchedulerElect, ret.SAScheduler.Name, ret.Namespace.Name)
+	rbacupdate.RoleBinding(ret.RBSchedulerAuth, ret.SAScheduler.Name, ret.Namespace.Name)
 	ret.DPScheduler.Namespace = ret.Namespace.Name
 	ret.ConfigMap.Namespace = ret.Namespace.Name
 
@@ -157,7 +168,9 @@ func (mf Manifests) ToObjects() []client.Object {
 		mf.CRScheduler,
 		mf.CRBScheduler,
 		mf.ConfigMap,
-		mf.RBScheduler,
+		mf.RSchedulerElect,
+		mf.RBSchedulerAuth,
+		mf.RBSchedulerElect,
 		mf.DPScheduler,
 		mf.SAController,
 		mf.CRController,
@@ -201,7 +214,15 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) {
 	if err != nil {
 		return mf, err
 	}
-	mf.RBScheduler, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, namespace)
+	mf.RSchedulerElect, err = manifests.Role(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, namespace)
+	if err != nil {
+		return mf, err
+	}
+	mf.RBSchedulerElect, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, "leaderelect", namespace)
+	if err != nil {
+		return mf, err
+	}
+	mf.RBSchedulerAuth, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginScheduler, "authread", namespace)
 	if err != nil {
 		return mf, err
 	}
@@ -222,7 +243,7 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) {
 	if err != nil {
 		return mf, err
 	}
-	mf.RBController, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginController, namespace)
+	mf.RBController, err = manifests.RoleBinding(manifests.ComponentSchedulerPlugin, manifests.SubComponentSchedulerPluginController, "", namespace)
 	if err != nil {
 		return mf, err
 	}
@@ -234,6 +255,27 @@ func GetManifests(plat platform.Platform, namespace string) (Manifests, error) {
 	return mf, nil
 }
 
+func leaderElectionParamsFromOpts(opts options.Scheduler) (*manifests.LeaderElectionParams, error) {
+	if !opts.LeaderElection {
+		return nil, nil
+	}
+	leap := manifests.LeaderElectionParams{
+		LeaderElect: true,
+	}
+	manifests.SetDefaultsLeaderElection(&leap)
+	var err error
+	tokens := strings.Split(opts.LeaderElectionResource, "/")
+	if len(tokens) == 1 {
+		leap.ResourceNamespace = tokens[0]
+	} else if len(tokens) == 2 {
+		leap.ResourceNamespace = tokens[0]
+		leap.ResourceName = tokens[1]
+	} else {
+		err = fmt.Errorf("malformed leader election resource: %q", opts.LeaderElectionResource)
+	}
+	return &leap, err
+}
+
 func newInt32(value int32) *int32 {
 	return &value
 }

pkg/manifests/yaml/sched/scheduler/clusterrole.yaml

@@ -6,20 +6,6 @@ rules:
 - apiGroups: ["", "events.k8s.io"]
   resources: ["events"]
   verbs: ["create", "patch", "update"]
-- apiGroups: ["coordination.k8s.io"]
-  resources: ["leases"]
-  verbs: ["create"]
-- apiGroups: ["coordination.k8s.io"]
-  resourceNames: ["kube-scheduler"]
-  resources: ["leases"]
-  verbs: ["get", "update"]
-- apiGroups: [""]
-  resources: ["endpoints"]
-  verbs: ["create"]
-- apiGroups: [""]
-  resourceNames: ["kube-scheduler"]
-  resources: ["endpoints"]
-  verbs: ["get", "update"]
 - apiGroups: [""]
   resources: ["nodes"]
   verbs: ["get", "list", "watch"]

pkg/manifests/yaml/sched/scheduler/role.yaml

@@ -0,0 +1,19 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: topology-aware-scheduler-leader-elect
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["create"]
+- apiGroups: ["coordination.k8s.io"]
+  resourceNames: ["nrtmatch-scheduler"]
+  resources: ["leases"]
+  verbs: ["get", "update"]
+- apiGroups: [""]
+  resources: ["endpoints"]
+  verbs: ["create"]
+- apiGroups: [""]
+  resourceNames: ["nrtmatch-scheduler"]
+  resources: ["endpoints"]
+  verbs: ["get", "update"]

pkg/manifests/yaml/sched/scheduler/rolebinding_leaderelect.yaml

@@ -0,0 +1,13 @@
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: topology-aware-scheduler-leader-elect
+  namespace: tas-scheduler
+subjects:
+  - kind: ServiceAccount
+    name: topology-aware-scheduler
+    namespace: tas-scheduler
+roleRef:
+  kind: Role
+  name: topology-aware-scheduler-leader-elect
+  apiGroup: rbac.authorization.k8s.io

pkg/objectwait/sched/sched.go

@@ -35,7 +35,9 @@ func Creatable(mf schedmf.Manifests, cli client.Client, log logr.Logger) []objec
 		{Obj: mf.SAScheduler},
 		{Obj: mf.CRScheduler},
 		{Obj: mf.CRBScheduler},
-		{Obj: mf.RBScheduler},
+		{Obj: mf.RSchedulerElect},
+		{Obj: mf.RBSchedulerElect},
+		{Obj: mf.RBSchedulerAuth},
 		{Obj: mf.ConfigMap},
 		{
 			Obj: mf.DPScheduler,
@@ -69,7 +71,9 @@ func Deletable(mf schedmf.Manifests, cli client.Client, log logr.Logger) []objec
 		// no need to remove objects created inside the namespace we just removed
 		{Obj: mf.CRBScheduler},
 		{Obj: mf.CRScheduler},
-		{Obj: mf.RBScheduler},
+		{Obj: mf.RBSchedulerAuth},
+		{Obj: mf.RBSchedulerElect},
+		{Obj: mf.RSchedulerElect},
 		{Obj: mf.CRBController},
 		{Obj: mf.CRController},
 		{Obj: mf.RBController},

pkg/options/options.go

@@ -61,6 +61,8 @@ type Scheduler struct {
 	PullIfNotPresent       bool
 	CacheResyncPeriod      time.Duration
 	CtrlPlaneAffinity      bool
+	LeaderElection         bool
+	LeaderElectionResource string
 	Verbose                int
 	ScoringStratConfigData string
 	CacheParamsConfigData  string