manifests: sched: support for LeaderElection params

add support to get/set LeaderEletion parameters
programmatically.

Signed-off-by: Francesco Romani <[email protected]>

pkg/manifests/sched/sched.go

@@ -39,9 +39,11 @@ import (
 )
 
 const (
-	DefaultProfileName  = "topology-aware-scheduler"
-	DefaultResyncPeriod = 5 * time.Second
-	DefaultVerbose      = 4
+	DefaultProfileName         = "topology-aware-scheduler"
+	DefaultResyncPeriod        = 5 * time.Second
+	DefaultVerbose             = 4
+	DefaultCtrlPlaneAffinity   = true
+	DefaultLeaderElectResource = manifests.LeaderElectionDefaultNamespace + "/" + manifests.LeaderElectionDefaultName
 )
 
 const (

pkg/manifests/schedparams.go

@@ -54,6 +54,11 @@ const (
 	ScoringStrategyLeastAllocated     = "LeastAllocated"
 )
 
+const (
+	LeaderElectionDefaultName      = "nrtmatch-scheduler"
+	LeaderElectionDefaultNamespace = "tas-scheduler"
+)
+
 func ValidateForeignPodsDetectMode(value string) error {
 	switch value {
 	case ForeignPodsDetectNone:
@@ -136,10 +141,26 @@ func ValidateScoringStrategyType(value string) error {
 	}
 }
 
+type LeaderElectionParams struct {
+	LeaderElect       bool   `json:"leaderElect"`
+	ResourceName      string `json:"resourceName,omitempty"`
+	ResourceNamespace string `json:"resourceNamespace,omitempty"`
+}
+
+func SetDefaultsLeaderElection(lep *LeaderElectionParams) {
+	if lep.ResourceName == "" {
+		lep.ResourceName = LeaderElectionDefaultName
+	}
+	if lep.ResourceNamespace == "" {
+		lep.ResourceNamespace = LeaderElectionDefaultNamespace
+	}
+}
+
 type ConfigParams struct {
 	ProfileName     string // can't be empty, so no need for pointer
 	Cache           *ConfigCacheParams
 	ScoringStrategy *ScoringStrategyParams
+	LeaderElection  *LeaderElectionParams
 }
 
 func DecodeSchedulerProfilesFromData(data []byte) ([]ConfigParams, error) {
@@ -151,6 +172,20 @@ func DecodeSchedulerProfilesFromData(data []byte) ([]ConfigParams, error) {
 		return params, nil
 	}
 
+	lead, ok, err := unstructured.NestedMap(r.Object, "leaderElection")
+	if err != nil {
+		klog.ErrorS(err, "failed to process unstructured data")
+		return params, err
+	}
+	var electParams *LeaderElectionParams
+	if ok {
+		electParams, err = extractLeaderElectionParams(lead)
+		if err != nil {
+			klog.ErrorS(err, "failed to extract leader election params")
+			return params, nil
+		}
+	}
+
 	profiles, ok, err := unstructured.NestedSlice(r.Object, "profiles")
 	if !ok || err != nil {
 		klog.ErrorS(err, "failed to process unstructured data", "profiles", ok)
@@ -200,6 +235,10 @@ func DecodeSchedulerProfilesFromData(data []byte) ([]ConfigParams, error) {
 				klog.ErrorS(err, "failed to extract params", "name", name, "profile", profileName)
 				continue
 			}
+			// since Leader Election Params is a global setting (independent from profiles),
+			// all profiles must share the same data. This is a modelization error which
+			// we should fix on later releases.
+			profileParams.LeaderElection = electParams
 
 			params = append(params, profileParams)
 		}
@@ -324,6 +363,34 @@ func extractParams(profileName string, args map[string]interface{}) (ConfigParam
 	return params, nil
 }
 
+func extractLeaderElectionParams(lead map[string]interface{}) (*LeaderElectionParams, error) {
+	var params LeaderElectionParams
+
+	enabled, ok, err := unstructured.NestedBool(lead, "leaderElect")
+	if !ok || err != nil {
+		return &params, fmt.Errorf("unexpected leaderElect data (err=%v)", err)
+	}
+	params.LeaderElect = enabled
+
+	resourceNamespace, ok, err := unstructured.NestedString(lead, "resourceNamespace")
+	if err != nil {
+		return &params, fmt.Errorf("unexpected resourceNamespace data: %w", err)
+	}
+	if ok {
+		params.ResourceNamespace = resourceNamespace
+	}
+
+	resourceName, ok, err := unstructured.NestedString(lead, "resourceName")
+	if err != nil {
+		return &params, fmt.Errorf("unexpected resourceName data: %w", err)
+	}
+	if ok {
+		params.ResourceName = resourceName
+	}
+
+	return &params, nil
+}
+
 func newInt64(v int64) *int64 {
 	return &v
 }

pkg/manifests/schedparams_test.go

@@ -33,10 +33,12 @@ func TestDecodeSchedulerConfigFromData(t *testing.T) {
 	}
 	testCases := []testCase{
 		{
-			name:           "nil",
-			data:           nil,
-			schedulerName:  "",
-			expectedParams: ConfigParams{},
+			name:          "nil",
+			data:          nil,
+			schedulerName: "",
+			expectedParams: ConfigParams{
+				LeaderElection: &LeaderElectionParams{},
+			},
 		},
 		{
 			name: "bad scheduler name",
@@ -60,8 +62,10 @@ profiles:
       - name: NodeResourceTopologyMatch
   schedulerName: topology-aware-scheduler
 `),
-			schedulerName:  "topo-aware-scheduler",
-			expectedParams: ConfigParams{},
+			schedulerName: "topo-aware-scheduler",
+			expectedParams: ConfigParams{
+				LeaderElection: &LeaderElectionParams{},
+			},
 		},
 		{
 			name: "bad scheduler params name",
@@ -85,8 +89,10 @@ profiles:
       - name: NodeResourceTopologyMatch
   schedulerName: topology-aware-scheduler
 `),
-			schedulerName:  "topology-aware-scheduler",
-			expectedParams: ConfigParams{},
+			schedulerName: "topology-aware-scheduler",
+			expectedParams: ConfigParams{
+				LeaderElection: &LeaderElectionParams{},
+			},
 		},
 		{
 			name: "empty params",
@@ -112,8 +118,9 @@ profiles:
 `),
 			schedulerName: "topology-aware-scheduler",
 			expectedParams: ConfigParams{
-				ProfileName: "topology-aware-scheduler",
-				Cache:       &ConfigCacheParams{},
+				ProfileName:    "topology-aware-scheduler",
+				Cache:          &ConfigCacheParams{},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -146,6 +153,7 @@ profiles:
 				Cache: &ConfigCacheParams{
 					ResyncPeriodSeconds: newInt64(5),
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -185,6 +193,7 @@ profiles:
 					ForeignPodsDetectMode: newString("None"),
 					InformerMode:          newString("Dedicated"),
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -220,6 +229,7 @@ profiles:
 					ResyncPeriodSeconds: newInt64(5),
 					ResyncMethod:        newString("OnlyExclusiveResources"),
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -253,6 +263,7 @@ profiles:
 				Cache: &ConfigCacheParams{
 					ForeignPodsDetectMode: newString("OnlyExclusiveResources"),
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -286,6 +297,7 @@ profiles:
 				Cache: &ConfigCacheParams{
 					InformerMode: newString("Shared"),
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -335,6 +347,7 @@ profiles:
 						},
 					},
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -369,6 +382,7 @@ profiles:
 				ScoringStrategy: &ScoringStrategyParams{
 					Type: "BalancedAllocation",
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -410,6 +424,7 @@ profiles:
 						},
 					},
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},
@@ -477,6 +492,7 @@ profiles:
 						},
 					},
 				},
+				LeaderElection: &LeaderElectionParams{},
 			},
 			expectedFound: true,
 		},

pkg/objectupdate/sched/render.go

@@ -62,6 +62,29 @@ func RenderConfig(data []byte, schedulerName string, params *manifests.ConfigPar
 
 	updated := false
 
+	if params.LeaderElection != nil {
+		lead, ok, err := unstructured.NestedMap(r.Object, "leaderElection")
+		if !ok || err != nil {
+			klog.ErrorS(err, "failed to process unstructured data", "leaderElection", ok)
+			return data, false, err
+		}
+
+		leadUpdated, err := updateLeaderElection(lead, params)
+		if err != nil {
+			klog.ErrorS(err, "failed to update unstructured data", "leaderElection", lead, "params", params)
+			return data, false, err
+		}
+		if leadUpdated {
+			updated = true
+		}
+
+		if err := unstructured.SetNestedMap(r.Object, lead, "leaderElection"); err != nil {
+			klog.ErrorS(err, "failed to override unstructured data", "data", "leaderElection")
+			return data, false, err
+		}
+
+	}
+
 	profiles, ok, err := unstructured.NestedSlice(r.Object, "profiles")
 	if !ok || err != nil {
 		klog.ErrorS(err, "failed to process unstructured data", "profiles", ok)
@@ -153,6 +176,32 @@ func RenderConfig(data []byte, schedulerName string, params *manifests.ConfigPar
 	return newData, updated, nil
 }
 
+func updateLeaderElection(lead map[string]interface{}, params *manifests.ConfigParams) (bool, error) {
+	var updated int
+	var err error
+
+	err = unstructured.SetNestedField(lead, params.LeaderElection.LeaderElect, "leaderElect")
+	if err != nil {
+		return updated > 0, err
+	}
+	updated++
+
+	err = unstructured.SetNestedField(lead, params.LeaderElection.ResourceName, "resourceName")
+	if err != nil {
+		return updated > 0, err
+	}
+	updated++
+
+	err = unstructured.SetNestedField(lead, params.LeaderElection.ResourceNamespace, "resourceNamespace")
+	if err != nil {
+		return updated > 0, err
+	}
+	updated++
+
+	return updated > 0, nil
+
+}
+
 func updateArgs(args map[string]interface{}, params *manifests.ConfigParams) (bool, error) {
 	var updated int
 	var err error