sched: support for informerMode option

The scheduler plugin gained support for the external informer, necessary to overcome mismatch between the scheduler and the ode views wrt the pods on nodes. Expose support in the deployer libraries to change this setting. Signed-off-by: Francesco Romani <[email protected]>
k8stopologyawareschedwg · Jan 30, 2024 · b6f4ef0 · b6f4ef0
1 parent e4574c1
commit b6f4ef0
Show file tree

Hide file tree

Showing 4 changed files with 111 additions and 28 deletions.
diff --git a/pkg/manifests/schedparams.go b/pkg/manifests/schedparams.go
@@ -42,6 +42,11 @@ const (
 	CacheResyncOnlyExclusiveResources = "OnlyExclusiveResources"
 )
 
+const (
+	CacheInformerShared    = "Shared"
+	CacheInformerDedicated = "Dedicated"
+)
+
 func ValidateForeignPodsDetectMode(value string) error {
 	switch value {
 	case ForeignPodsDetectNone:
@@ -68,10 +73,22 @@ func ValidateCacheResyncMethod(value string) error {
 	}
 }
 
+func ValidateCacheInformerMode(value string) error {
+	switch value {
+	case CacheInformerShared:
+		return nil
+	case CacheInformerDedicated:
+		return nil
+	default:
+		return fmt.Errorf("unsupported cacheInformerMode: %v", value)
+	}
+}
+
 type ConfigCacheParams struct {
 	ResyncPeriodSeconds   *int64
 	ResyncMethod          *string
 	ForeignPodsDetectMode *string
+	InformerMode          *string
 }
 
 type ConfigParams struct {
@@ -196,6 +213,17 @@ func extractParams(profileName string, args map[string]interface{}) (ConfigParam
 			}
 			params.Cache.ForeignPodsDetectMode = &foreignPodsDetect
 		}
+
+		informerMode, cacheOk, err := unstructured.NestedString(cacheArgs, "informerMode")
+		if err != nil {
+			return params, fmt.Errorf("cannot process field cache.informerMode: %w", err)
+		}
+		if cacheOk {
+			if err := ValidateCacheInformerMode(informerMode); err != nil {
+				return params, err
+			}
+			params.Cache.InformerMode = &informerMode
+		}
 	}
 	return params, nil
 }
diff --git a/pkg/manifests/schedparams_test.go b/pkg/manifests/schedparams_test.go
@@ -161,6 +161,7 @@ profiles:
       cache:
         foreignPodsDetect: None
         resyncMethod: Autodetect
+        informerMode: Dedicated
       cacheResyncPeriodSeconds: 5
     name: NodeResourceTopologyMatch
   plugins:
@@ -182,6 +183,7 @@ profiles:
 					ResyncPeriodSeconds:   newInt64(5),
 					ResyncMethod:          newString("Autodetect"),
 					ForeignPodsDetectMode: newString("None"),
+					InformerMode:          newString("Dedicated"),
 				},
 			},
 			expectedFound: true,
@@ -222,7 +224,7 @@ profiles:
 			expectedFound: true,
 		},
 		{
-			name: "zero resync period and some cache params",
+			name: "zero resync period and some cache params - 2",
 			data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3
 kind: KubeSchedulerConfiguration
 leaderElection:
@@ -254,6 +256,39 @@ profiles:
 			},
 			expectedFound: true,
 		},
+		{
+			name: "zero resync period and some cache params - 3",
+			data: []byte(`apiVersion: kubescheduler.config.k8s.io/v1beta3
+kind: KubeSchedulerConfiguration
+leaderElection:
+  leaderElect: false
+profiles:
+- pluginConfig:
+  - args:
+      cache:
+        informerMode: Shared
+    name: NodeResourceTopologyMatch
+  plugins:
+    filter:
+      enabled:
+      - name: NodeResourceTopologyMatch
+    reserve:
+      enabled:
+      - name: NodeResourceTopologyMatch
+    score:
+      enabled:
+      - name: NodeResourceTopologyMatch
+  schedulerName: topology-aware-scheduler
+`),
+			schedulerName: "topology-aware-scheduler",
+			expectedParams: ConfigParams{
+				ProfileName: "topology-aware-scheduler",
+				Cache: &ConfigCacheParams{
+					InformerMode: newString("Shared"),
+				},
+			},
+			expectedFound: true,
+		},
 	}
 
 	for _, tc := range testCases {

diff --git a/pkg/objectupdate/sched/render.go b/pkg/objectupdate/sched/render.go
@@ -176,9 +176,12 @@ func updateArgs(args map[string]interface{}, params *manifests.ConfigParams) (bo
 		return updated > 0, err
 	}
 
-	cacheArgsUpdated, err := updateCacheArgs(cacheArgs, params)
-	if err != nil {
-		return updated > 0, err
+	var cacheArgsUpdated int
+	if params.Cache != nil {
+		cacheArgsUpdated, err = updateCacheArgs(cacheArgs, params)
+		if err != nil {
+			return updated > 0, err
+		}
 	}
 	updated += cacheArgsUpdated
 
@@ -194,31 +197,41 @@ func updateCacheArgs(args map[string]interface{}, params *manifests.ConfigParams
 	var updated int
 	var err error
 
-	if params.Cache != nil {
-		if params.Cache.ResyncMethod != nil {
-			resyncMethod := *params.Cache.ResyncMethod // shortcut
-			err = manifests.ValidateCacheResyncMethod(resyncMethod)
-			if err != nil {
-				return updated, err
-			}
-			err = unstructured.SetNestedField(args, resyncMethod, "resyncMethod")
-			if err != nil {
-				return updated, err
-			}
-			updated++
+	if params.Cache.ResyncMethod != nil {
+		resyncMethod := *params.Cache.ResyncMethod // shortcut
+		err = manifests.ValidateCacheResyncMethod(resyncMethod)
+		if err != nil {
+			return updated, err
 		}
-		if params.Cache.ForeignPodsDetectMode != nil {
-			foreignPodsMode := *params.Cache.ForeignPodsDetectMode // shortcut
-			err = manifests.ValidateForeignPodsDetectMode(foreignPodsMode)
-			if err != nil {
-				return updated, err
-			}
-			err = unstructured.SetNestedField(args, foreignPodsMode, "foreignPodsDetect")
-			if err != nil {
-				return updated, err
-			}
-			updated++
+		err = unstructured.SetNestedField(args, resyncMethod, "resyncMethod")
+		if err != nil {
+			return updated, err
+		}
+		updated++
+	}
+	if params.Cache.ForeignPodsDetectMode != nil {
+		foreignPodsMode := *params.Cache.ForeignPodsDetectMode // shortcut
+		err = manifests.ValidateForeignPodsDetectMode(foreignPodsMode)
+		if err != nil {
+			return updated, err
+		}
+		err = unstructured.SetNestedField(args, foreignPodsMode, "foreignPodsDetect")
+		if err != nil {
+			return updated, err
+		}
+		updated++
+	}
+	if params.Cache.InformerMode != nil {
+		informerMode := *params.Cache.InformerMode // shortcut
+		err = manifests.ValidateCacheInformerMode(informerMode)
+		if err != nil {
+			return updated, err
+		}
+		err = unstructured.SetNestedField(args, informerMode, "informerMode")
+		if err != nil {
+			return updated, err
 		}
+		updated++
 	}
 
 	return updated, nil

diff --git a/pkg/objectupdate/sched/render_test.go b/pkg/objectupdate/sched/render_test.go
@@ -19,6 +19,7 @@ package sched
 import (
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/k8stopologyawareschedwg/deployer/pkg/manifests"
 )
 
@@ -220,6 +221,7 @@ profiles:
 					ResyncPeriodSeconds:   newInt64(42),
 					ResyncMethod:          newString("OnlyExclusiveResources"),
 					ForeignPodsDetectMode: newString("OnlyExclusiveResources"),
+					InformerMode:          newString("Dedicated"),
 				},
 			},
 			initial: configTemplateEmpty,
@@ -232,6 +234,7 @@ profiles:
   - args:
       cache:
         foreignPodsDetect: OnlyExclusiveResources
+        informerMode: Dedicated
         resyncMethod: OnlyExclusiveResources
       cacheResyncPeriodSeconds: 42
     name: NodeResourceTopologyMatch
@@ -292,6 +295,7 @@ profiles:
 					ResyncPeriodSeconds:   newInt64(7),
 					ResyncMethod:          newString("Autodetect"),
 					ForeignPodsDetectMode: newString("None"),
+					InformerMode:          newString("Shared"),
 				},
 			},
 			initial: configTemplateAllValuesFineTuned,
@@ -304,6 +308,7 @@ profiles:
   - args:
       cache:
         foreignPodsDetect: None
+        informerMode: Shared
         resyncMethod: Autodetect
       cacheResyncPeriodSeconds: 7
     name: NodeResourceTopologyMatch
@@ -372,6 +377,7 @@ profiles:
   - args:
       cache:
         foreignPodsDetect: All
+        informerMode: Dedicated
         resyncMethod: OnlyExclusiveResources
       cacheResyncPeriodSeconds: 5
     name: NodeResourceTopologyMatch
@@ -409,7 +415,7 @@ profiles:
 
 			rendered := string(data)
 			if rendered != tc.expected {
-				t.Errorf("rendering failed.\nrendered=[%s]\nexpected=[%s]\n", rendered, tc.expected)
+				t.Errorf("rendering failed.\nrendered=[%s]\nexpected=[%s]\ndiff=[%s]\n", rendered, tc.expected, cmp.Diff(rendered, tc.expected))
 			}
 		})
 	}
@@ -467,6 +473,7 @@ profiles:
   - args:
       cache:
         foreignPodsDetect: OnlyExclusiveResources
+        informerMode: Dedicated
         resyncMethod: OnlyExclusiveResources
       cacheResyncPeriodSeconds: 5
     name: NodeResourceTopologyMatch