From 4a6279f615248eddfcf423e60dde2233a1eb6818 Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Sun, 11 Apr 2021 15:19:23 +0800 Subject: [PATCH 1/6] chore: add IDE and OS paraphernalia to gitignore --- .gitignore | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index de1ad0e7e9..475be01b06 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,10 @@ # Ignore binaries built by the makefile to avoid accidentally committing them /images/kubefed/hyperfed /images/kubefed/controller-manager + +# editor and IDE paraphernalia +.idea +.vscode + +# macOS paraphernalia +.DS_Store From 65a18ead058e60eefb1ff06959d4986b1acc8b09 Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Sun, 11 Apr 2021 20:21:13 +0800 Subject: [PATCH 2/6] introduce concurrent option to worker --- .../federatedtypeconfig/controller.go | 2 +- .../schedulingmanager/controller.go | 2 +- .../schedulingpreference/controller.go | 6 ++- pkg/controller/status/controller.go | 6 ++- pkg/controller/sync/controller.go | 6 ++- pkg/controller/util/worker.go | 48 ++++++++++++------- pkg/controller/util/worker_test.go | 2 +- 7 files changed, 47 insertions(+), 25 deletions(-) diff --git a/pkg/controller/federatedtypeconfig/controller.go b/pkg/controller/federatedtypeconfig/controller.go index 0e6cade6d0..7d6cc5f558 100644 --- a/pkg/controller/federatedtypeconfig/controller.go +++ b/pkg/controller/federatedtypeconfig/controller.go @@ -90,7 +90,7 @@ func newController(config *util.ControllerConfig) (*Controller, error) { stopChannels: make(map[string]chan struct{}), } - c.worker = util.NewReconcileWorker("federatedtypeconfig", c.reconcile, util.WorkerTiming{}) + c.worker = util.NewReconcileWorker("federatedtypeconfig", c.reconcile, util.WorkerOptions{}) // Only watch the KubeFed namespace to ensure // restrictive authz can be applied to a namespaced diff --git a/pkg/controller/schedulingmanager/controller.go b/pkg/controller/schedulingmanager/controller.go index 4c8881ae3e..afab17b2a3 100644 --- a/pkg/controller/schedulingmanager/controller.go +++ b/pkg/controller/schedulingmanager/controller.go @@ -92,7 +92,7 @@ func newSchedulingManager(config *util.ControllerConfig) (*SchedulingManager, er schedulers: util.NewSafeMap(), } - c.worker = util.NewReconcileWorker("schedulingmanager", c.reconcile, util.WorkerTiming{}) + c.worker = util.NewReconcileWorker("schedulingmanager", c.reconcile, util.WorkerOptions{}) var err error c.store, c.controller, err = util.NewGenericInformer( diff --git a/pkg/controller/schedulingpreference/controller.go b/pkg/controller/schedulingpreference/controller.go index 78dfce1e5a..cb119c33db 100644 --- a/pkg/controller/schedulingpreference/controller.go +++ b/pkg/controller/schedulingpreference/controller.go @@ -105,8 +105,10 @@ func newSchedulingPreferenceController(config *util.ControllerConfig, scheduling eventRecorder: recorder, } - s.worker = util.NewReconcileWorker(strings.ToLower(schedulingType.Kind), s.reconcile, util.WorkerTiming{ - ClusterSyncDelay: s.clusterAvailableDelay, + s.worker = util.NewReconcileWorker(strings.ToLower(schedulingType.Kind), s.reconcile, util.WorkerOptions{ + WorkerTiming: util.WorkerTiming{ + ClusterSyncDelay: s.clusterAvailableDelay, + }, }) eventHandlers := schedulingtypes.SchedulerEventHandlers{ diff --git a/pkg/controller/status/controller.go b/pkg/controller/status/controller.go index a38252b7f2..ebc33f4886 100644 --- a/pkg/controller/status/controller.go +++ b/pkg/controller/status/controller.go @@ -123,8 +123,10 @@ func newKubeFedStatusController(controllerConfig *util.ControllerConfig, typeCon fedNamespace: controllerConfig.KubeFedNamespace, } - s.worker = util.NewReconcileWorker(strings.ToLower(statusAPIResource.Kind), s.reconcile, util.WorkerTiming{ - ClusterSyncDelay: s.clusterAvailableDelay, + s.worker = util.NewReconcileWorker(strings.ToLower(statusAPIResource.Kind), s.reconcile, util.WorkerOptions{ + WorkerTiming: util.WorkerTiming{ + ClusterSyncDelay: s.clusterAvailableDelay, + }, }) // Build deliverer for triggering cluster reconciliations. diff --git a/pkg/controller/sync/controller.go b/pkg/controller/sync/controller.go index 6a94f680c7..4cf5351241 100644 --- a/pkg/controller/sync/controller.go +++ b/pkg/controller/sync/controller.go @@ -132,8 +132,10 @@ func newKubeFedSyncController(controllerConfig *util.ControllerConfig, typeConfi rawResourceStatusCollection: controllerConfig.RawResourceStatusCollection, } - s.worker = util.NewReconcileWorker(strings.ToLower(federatedTypeAPIResource.Kind), s.reconcile, util.WorkerTiming{ - ClusterSyncDelay: s.clusterAvailableDelay, + s.worker = util.NewReconcileWorker(strings.ToLower(federatedTypeAPIResource.Kind), s.reconcile, util.WorkerOptions{ + WorkerTiming: util.WorkerTiming{ + ClusterSyncDelay: s.clusterAvailableDelay, + }, }) // Build deliverer for triggering cluster reconciliations. diff --git a/pkg/controller/util/worker.go b/pkg/controller/util/worker.go index d38ddc9295..e972c93108 100644 --- a/pkg/controller/util/worker.go +++ b/pkg/controller/util/worker.go @@ -38,6 +38,13 @@ type ReconcileWorker interface { SetDelay(retryDelay, clusterSyncDelay time.Duration) } +type WorkerOptions struct { + WorkerTiming + + // MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1. + MaxConcurrentReconciles int +} + type WorkerTiming struct { Interval time.Duration RetryDelay time.Duration @@ -53,6 +60,8 @@ type asyncWorker struct { timing WorkerTiming + maxConcurrentReconciles int + // For triggering reconciliation of a single resource. This is // used when there is an add/update/delete operation on a resource // in either the API of the cluster hosting KubeFed or in the API @@ -66,26 +75,30 @@ type asyncWorker struct { backoff *flowcontrol.Backoff } -func NewReconcileWorker(name string, reconcile ReconcileFunc, timing WorkerTiming) ReconcileWorker { - if timing.Interval == 0 { - timing.Interval = time.Second * 1 +func NewReconcileWorker(name string, reconcile ReconcileFunc, options WorkerOptions) ReconcileWorker { + if options.Interval == 0 { + options.Interval = time.Second * 1 } - if timing.RetryDelay == 0 { - timing.RetryDelay = time.Second * 10 + if options.RetryDelay == 0 { + options.RetryDelay = time.Second * 10 } - if timing.InitialBackoff == 0 { - timing.InitialBackoff = time.Second * 5 + if options.InitialBackoff == 0 { + options.InitialBackoff = time.Second * 5 } - if timing.MaxBackoff == 0 { - timing.MaxBackoff = time.Minute + if options.MaxBackoff == 0 { + options.MaxBackoff = time.Minute + } + if options.MaxConcurrentReconciles == 0 { + options.MaxConcurrentReconciles = 1 } return &asyncWorker{ - name: name, - reconcile: reconcile, - timing: timing, - deliverer: NewDelayingDeliverer(), - queue: workqueue.NewNamed(name), - backoff: flowcontrol.NewBackOff(timing.InitialBackoff, timing.MaxBackoff), + name: name, + reconcile: reconcile, + timing: options.WorkerTiming, + maxConcurrentReconciles: options.MaxConcurrentReconciles, + deliverer: NewDelayingDeliverer(), + queue: workqueue.NewNamed(name), + backoff: flowcontrol.NewBackOff(options.InitialBackoff, options.MaxBackoff), } } @@ -122,7 +135,10 @@ func (w *asyncWorker) Run(stopChan <-chan struct{}) { w.queue.Add(*qualifiedName) } }) - go wait.Until(w.worker, w.timing.Interval, stopChan) + + for i := 0; i < w.maxConcurrentReconciles; i++ { + go wait.Until(w.worker, w.timing.Interval, stopChan) + } // Ensure all goroutines are cleaned up when the stop channel closes go func() { diff --git a/pkg/controller/util/worker_test.go b/pkg/controller/util/worker_test.go index 1d2c113859..a0fe84ca54 100644 --- a/pkg/controller/util/worker_test.go +++ b/pkg/controller/util/worker_test.go @@ -61,7 +61,7 @@ func TestDeduplicate(t *testing.T) { }) return StatusAllOK }, - WorkerTiming{}, + WorkerOptions{}, ) // run worker From ec61e87c386bae01a1603fe943073874efbe6481 Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Tue, 4 May 2021 20:36:44 +0800 Subject: [PATCH 3/6] introduce controller concurrency settings to KubeFedConfig --- .../app/controller-manager.go | 3 +++ config/kubefedconfig.yaml | 3 +++ pkg/apis/core/v1beta1/defaults/defaults.go | 11 ++++++++++ .../core/v1beta1/defaults/defaults_test.go | 15 ++++++++++++++ pkg/apis/core/v1beta1/kubefedconfig_types.go | 13 ++++++++++++ .../core/v1beta1/validation/validation.go | 9 +++++++++ .../v1beta1/validation/validation_test.go | 20 +++++++++++++++++++ pkg/controller/status/controller.go | 1 + pkg/controller/sync/controller.go | 1 + pkg/controller/testdata/fixtures/crds.yaml | 13 ++++++++++++ pkg/controller/util/controllerconfig.go | 14 +++++++------ 11 files changed, 97 insertions(+), 6 deletions(-) diff --git a/cmd/controller-manager/app/controller-manager.go b/cmd/controller-manager/app/controller-manager.go index bef051d52f..f28c4c3764 100644 --- a/cmd/controller-manager/app/controller-manager.go +++ b/cmd/controller-manager/app/controller-manager.go @@ -365,6 +365,9 @@ func setOptionsByKubeFedConfig(opts *options.Options) { opts.ClusterHealthCheckConfig.FailureThreshold = *spec.ClusterHealthCheck.FailureThreshold opts.ClusterHealthCheckConfig.SuccessThreshold = *spec.ClusterHealthCheck.SuccessThreshold + opts.Config.MaxConcurrentSyncReconciles = *spec.SyncController.MaxConcurrentReconciles + opts.Config.MaxConcurrentStatusReconciles = *spec.StatusController.MaxConcurrentReconciles + opts.Config.SkipAdoptingResources = *spec.SyncController.AdoptResources == corev1b1.AdoptResourcesDisabled var featureGates = make(map[string]bool) diff --git a/config/kubefedconfig.yaml b/config/kubefedconfig.yaml index 1fcb77dfed..777022dc2b 100644 --- a/config/kubefedconfig.yaml +++ b/config/kubefedconfig.yaml @@ -26,4 +26,7 @@ spec: successThreshold: 1 timeout: 3s syncController: + maxConcurrentReconciles: 1 adoptResources: Enabled + statusController: + maxConcurrentReconciles: 1 diff --git a/pkg/apis/core/v1beta1/defaults/defaults.go b/pkg/apis/core/v1beta1/defaults/defaults.go index 31ce87d051..1abb5fc11f 100644 --- a/pkg/apis/core/v1beta1/defaults/defaults.go +++ b/pkg/apis/core/v1beta1/defaults/defaults.go @@ -39,6 +39,9 @@ const ( DefaultClusterHealthCheckFailureThreshold = 3 DefaultClusterHealthCheckSuccessThreshold = 1 DefaultClusterHealthCheckTimeout = 3 * time.Second + + DefaultSyncControllerMaxConcurrentReconciles = 1 + DefaultStatusControllerMaxConcurrentReconciles = 1 ) func SetDefaultKubeFedConfig(fedConfig *v1beta1.KubeFedConfig) { @@ -87,10 +90,18 @@ func SetDefaultKubeFedConfig(fedConfig *v1beta1.KubeFedConfig) { spec.SyncController = &v1beta1.SyncControllerConfig{} } + setInt64(&spec.SyncController.MaxConcurrentReconciles, DefaultSyncControllerMaxConcurrentReconciles) + if spec.SyncController.AdoptResources == nil { spec.SyncController.AdoptResources = new(v1beta1.ResourceAdoption) *spec.SyncController.AdoptResources = v1beta1.AdoptResourcesEnabled } + + if spec.StatusController == nil { + spec.StatusController = &v1beta1.StatusControllerConfig{} + } + + setInt64(&spec.StatusController.MaxConcurrentReconciles, DefaultStatusControllerMaxConcurrentReconciles) } func setDefaultKubeFedFeatureGates(fgc []v1beta1.FeatureGatesConfig) []v1beta1.FeatureGatesConfig { diff --git a/pkg/apis/core/v1beta1/defaults/defaults_test.go b/pkg/apis/core/v1beta1/defaults/defaults_test.go index 8456395da8..2975917b3d 100644 --- a/pkg/apis/core/v1beta1/defaults/defaults_test.go +++ b/pkg/apis/core/v1beta1/defaults/defaults_test.go @@ -120,12 +120,27 @@ func TestSetDefaultKubeFedConfig(t *testing.T) { successCases["spec.clusterHealthCheck.timeout is preserved"] = KubeFedConfigComparison{timeoutKFC, modifiedTimeoutKFC} // SyncController + syncControllerMaxConcurrentReconcilesKFC := defaultKubeFedConfig() + syncControllerMaxConcurrentReconciles := int64(DefaultSyncControllerMaxConcurrentReconciles + 3) + syncControllerMaxConcurrentReconcilesKFC.Spec.SyncController.MaxConcurrentReconciles = &syncControllerMaxConcurrentReconciles + modifiedSyncControllerMaxConcurrentReconcilesKFC := syncControllerMaxConcurrentReconcilesKFC.DeepCopyObject().(*v1beta1.KubeFedConfig) + SetDefaultKubeFedConfig(modifiedSyncControllerMaxConcurrentReconcilesKFC) + successCases["spec.syncController.maxConcurrentReconciles is preserved"] = KubeFedConfigComparison{syncControllerMaxConcurrentReconcilesKFC, modifiedSyncControllerMaxConcurrentReconcilesKFC} + adoptResourcesKFC := defaultKubeFedConfig() *adoptResourcesKFC.Spec.SyncController.AdoptResources = v1beta1.AdoptResourcesDisabled modifiedAdoptResourcesKFC := adoptResourcesKFC.DeepCopyObject().(*v1beta1.KubeFedConfig) SetDefaultKubeFedConfig(modifiedAdoptResourcesKFC) successCases["spec.leaderElect.adoptResources is preserved"] = KubeFedConfigComparison{adoptResourcesKFC, modifiedAdoptResourcesKFC} + // StatusController + statusControllerMaxConcurrentReconcilesKFC := defaultKubeFedConfig() + statusControllerMaxConcurrentReconciles := int64(DefaultStatusControllerMaxConcurrentReconciles + 3) + statusControllerMaxConcurrentReconcilesKFC.Spec.StatusController.MaxConcurrentReconciles = &statusControllerMaxConcurrentReconciles + modifiedStatusControllerMaxConcurrentReconcilesKFC := statusControllerMaxConcurrentReconcilesKFC.DeepCopyObject().(*v1beta1.KubeFedConfig) + SetDefaultKubeFedConfig(modifiedStatusControllerMaxConcurrentReconcilesKFC) + successCases["spec.statusController.maxConcurrentReconciles is preserved"] = KubeFedConfigComparison{statusControllerMaxConcurrentReconcilesKFC, modifiedStatusControllerMaxConcurrentReconcilesKFC} + for k, v := range successCases { if !reflect.DeepEqual(v.original, v.modified) { t.Errorf("[%s] expected success: original=%+v, modified=%+v", k, *v.original, *v.modified) diff --git a/pkg/apis/core/v1beta1/kubefedconfig_types.go b/pkg/apis/core/v1beta1/kubefedconfig_types.go index b86930b91e..73a201b986 100644 --- a/pkg/apis/core/v1beta1/kubefedconfig_types.go +++ b/pkg/apis/core/v1beta1/kubefedconfig_types.go @@ -37,6 +37,8 @@ type KubeFedConfigSpec struct { ClusterHealthCheck *ClusterHealthCheckConfig `json:"clusterHealthCheck,omitempty"` // +optional SyncController *SyncControllerConfig `json:"syncController,omitempty"` + // +optional + StatusController *StatusControllerConfig `json:"statusController,omitempty"` } type DurationConfig struct { @@ -105,6 +107,10 @@ type ClusterHealthCheckConfig struct { } type SyncControllerConfig struct { + // The maximum number of concurrent Reconciles of sync controller which can be run. + // Defaults to 1. + // +optional + MaxConcurrentReconciles *int64 `json:"maxConcurrentReconciles,omitempty"` // Whether to adopt pre-existing resources in member clusters. Defaults to // "Enabled". // +optional @@ -118,6 +124,13 @@ const ( AdoptResourcesDisabled ResourceAdoption = "Disabled" ) +type StatusControllerConfig struct { + // The maximum number of concurrent Reconciles of status controller which can be run. + // Defaults to 1. + // +optional + MaxConcurrentReconciles *int64 `json:"maxConcurrentReconciles,omitempty"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:path=kubefedconfigs diff --git a/pkg/apis/core/v1beta1/validation/validation.go b/pkg/apis/core/v1beta1/validation/validation.go index 45ef4b19ea..b63145f9ec 100644 --- a/pkg/apis/core/v1beta1/validation/validation.go +++ b/pkg/apis/core/v1beta1/validation/validation.go @@ -356,10 +356,19 @@ func ValidateKubeFedConfig(kubeFedConfig, oldKubeFedConfig *v1beta1.KubeFedConfi case sync.AdoptResources == nil: allErrs = append(allErrs, field.Required(adoptPath, "")) default: + allErrs = append(allErrs, validateIntPtrGreaterThan0(syncPath.Child("maxConcurrentReconciles"), sync.MaxConcurrentReconciles)...) allErrs = append(allErrs, validateEnumStrings(adoptPath, string(*sync.AdoptResources), []string{string(v1beta1.AdoptResourcesEnabled), string(v1beta1.AdoptResourcesDisabled)})...) } + statusController := spec.StatusController + statusControllerPath := specPath.Child("statusController") + if statusController == nil { + allErrs = append(allErrs, field.Required(statusControllerPath, "")) + } else { + allErrs = append(allErrs, validateIntPtrGreaterThan0(statusControllerPath.Child("maxConcurrentReconciles"), statusController.MaxConcurrentReconciles)...) + } + return allErrs } diff --git a/pkg/apis/core/v1beta1/validation/validation_test.go b/pkg/apis/core/v1beta1/validation/validation_test.go index a6e6a3505e..23fa73eb16 100644 --- a/pkg/apis/core/v1beta1/validation/validation_test.go +++ b/pkg/apis/core/v1beta1/validation/validation_test.go @@ -844,6 +844,14 @@ func TestValidateKubeFedConfig(t *testing.T) { invalidSyncControllerNil.Spec.SyncController = nil errorCases["spec.syncController: Required value"] = invalidSyncControllerNil + invalidSyncControllerMaxConcurrentReconcilesNil := testcommon.ValidKubeFedConfig() + invalidSyncControllerMaxConcurrentReconcilesNil.Spec.SyncController.MaxConcurrentReconciles = nil + errorCases["spec.syncController.maxConcurrentReconciles: Required value"] = invalidSyncControllerMaxConcurrentReconcilesNil + + invalidSyncControllerMaxConcurrentReconcilesGreaterThan0 := testcommon.ValidKubeFedConfig() + invalidSyncControllerMaxConcurrentReconcilesGreaterThan0.Spec.SyncController.MaxConcurrentReconciles = zeroIntPtr + errorCases["spec.syncController.maxConcurrentReconciles: Invalid value"] = invalidSyncControllerMaxConcurrentReconcilesGreaterThan0 + invalidAdoptResourcesNil := testcommon.ValidKubeFedConfig() invalidAdoptResourcesNil.Spec.SyncController.AdoptResources = nil errorCases["spec.syncController.adoptResources: Required value"] = invalidAdoptResourcesNil @@ -853,6 +861,18 @@ func TestValidateKubeFedConfig(t *testing.T) { invalidAdoptResources.Spec.SyncController.AdoptResources = &invalidAdoptResourcesValue errorCases["spec.syncController.adoptResources: Unsupported value"] = invalidAdoptResources + invalidStatusControllerNil := testcommon.ValidKubeFedConfig() + invalidStatusControllerNil.Spec.StatusController = nil + errorCases["spec.statusController: Required value"] = invalidStatusControllerNil + + invalidStatusControllerMaxConcurrentReconcilesNil := testcommon.ValidKubeFedConfig() + invalidStatusControllerMaxConcurrentReconcilesNil.Spec.StatusController.MaxConcurrentReconciles = nil + errorCases["spec.statusController.maxConcurrentReconciles: Required value"] = invalidStatusControllerMaxConcurrentReconcilesNil + + invalidStatusControllerMaxConcurrentReconcilesGreaterThan0 := testcommon.ValidKubeFedConfig() + invalidStatusControllerMaxConcurrentReconcilesGreaterThan0.Spec.StatusController.MaxConcurrentReconciles = zeroIntPtr + errorCases["spec.statusController.maxConcurrentReconciles: Invalid value"] = invalidStatusControllerMaxConcurrentReconcilesGreaterThan0 + for k, v := range errorCases { errs := ValidateKubeFedConfig(v, testcommon.ValidKubeFedConfig()) if len(errs) == 0 { diff --git a/pkg/controller/status/controller.go b/pkg/controller/status/controller.go index ebc33f4886..ada8dcd211 100644 --- a/pkg/controller/status/controller.go +++ b/pkg/controller/status/controller.go @@ -127,6 +127,7 @@ func newKubeFedStatusController(controllerConfig *util.ControllerConfig, typeCon WorkerTiming: util.WorkerTiming{ ClusterSyncDelay: s.clusterAvailableDelay, }, + MaxConcurrentReconciles: int(controllerConfig.MaxConcurrentStatusReconciles), }) // Build deliverer for triggering cluster reconciliations. diff --git a/pkg/controller/sync/controller.go b/pkg/controller/sync/controller.go index 4cf5351241..1d5fd9230a 100644 --- a/pkg/controller/sync/controller.go +++ b/pkg/controller/sync/controller.go @@ -136,6 +136,7 @@ func newKubeFedSyncController(controllerConfig *util.ControllerConfig, typeConfi WorkerTiming: util.WorkerTiming{ ClusterSyncDelay: s.clusterAvailableDelay, }, + MaxConcurrentReconciles: int(controllerConfig.MaxConcurrentSyncReconciles), }) // Build deliverer for triggering cluster reconciliations. diff --git a/pkg/controller/testdata/fixtures/crds.yaml b/pkg/controller/testdata/fixtures/crds.yaml index 99321a4de9..2db7f5f246 100644 --- a/pkg/controller/testdata/fixtures/crds.yaml +++ b/pkg/controller/testdata/fixtures/crds.yaml @@ -774,12 +774,25 @@ spec: `Namespaced` or `Cluster`. `Namespaced` indicates that the KubeFed namespace will be the only target of the control plane. type: string + statusController: + properties: + maxConcurrentReconciles: + description: The maximum number of concurrent Reconciles of status + controller which can be run. Defaults to 1. + format: int64 + type: integer + type: object syncController: properties: adoptResources: description: Whether to adopt pre-existing resources in member clusters. Defaults to "Enabled". type: string + maxConcurrentReconciles: + description: The maximum number of concurrent Reconciles of sync + controller which can be run. Defaults to 1. + format: int64 + type: integer type: object required: - scope diff --git a/pkg/controller/util/controllerconfig.go b/pkg/controller/util/controllerconfig.go index af09f46971..1731711960 100644 --- a/pkg/controller/util/controllerconfig.go +++ b/pkg/controller/util/controllerconfig.go @@ -68,12 +68,14 @@ type ClusterHealthCheckConfig struct { // controllers. type ControllerConfig struct { KubeFedNamespaces - KubeConfig *restclient.Config - ClusterAvailableDelay time.Duration - ClusterUnavailableDelay time.Duration - MinimizeLatency bool - SkipAdoptingResources bool - RawResourceStatusCollection bool + KubeConfig *restclient.Config + ClusterAvailableDelay time.Duration + ClusterUnavailableDelay time.Duration + MinimizeLatency bool + MaxConcurrentSyncReconciles int64 + MaxConcurrentStatusReconciles int64 + SkipAdoptingResources bool + RawResourceStatusCollection bool } func (c *ControllerConfig) LimitedScope() bool { From 58e12e984318840611d27ff24b8fb418ad972d0b Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Tue, 4 May 2021 19:34:40 +0800 Subject: [PATCH 4/6] update generated code --- .../core/v1beta1/zz_generated.deepcopy.go | 30 +++++++++++++++++++ test/common/bindata.go | 3 ++ 2 files changed, 33 insertions(+) diff --git a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go index 694be4fa27..013f3a6c77 100644 --- a/pkg/apis/core/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/core/v1beta1/zz_generated.deepcopy.go @@ -454,6 +454,11 @@ func (in *KubeFedConfigSpec) DeepCopyInto(out *KubeFedConfigSpec) { *out = new(SyncControllerConfig) (*in).DeepCopyInto(*out) } + if in.StatusController != nil { + in, out := &in.StatusController, &out.StatusController + *out = new(StatusControllerConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeFedConfigSpec. @@ -516,9 +521,34 @@ func (in *LocalSecretReference) DeepCopy() *LocalSecretReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StatusControllerConfig) DeepCopyInto(out *StatusControllerConfig) { + *out = *in + if in.MaxConcurrentReconciles != nil { + in, out := &in.MaxConcurrentReconciles, &out.MaxConcurrentReconciles + *out = new(int64) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatusControllerConfig. +func (in *StatusControllerConfig) DeepCopy() *StatusControllerConfig { + if in == nil { + return nil + } + out := new(StatusControllerConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SyncControllerConfig) DeepCopyInto(out *SyncControllerConfig) { *out = *in + if in.MaxConcurrentReconciles != nil { + in, out := &in.MaxConcurrentReconciles, &out.MaxConcurrentReconciles + *out = new(int64) + **out = **in + } if in.AdoptResources != nil { in, out := &in.AdoptResources, &out.AdoptResources *out = new(ResourceAdoption) diff --git a/test/common/bindata.go b/test/common/bindata.go index 51e63f26db..ef4385ee64 100644 --- a/test/common/bindata.go +++ b/test/common/bindata.go @@ -400,7 +400,10 @@ spec: successThreshold: 1 timeout: 3s syncController: + maxConcurrentReconciles: 1 adoptResources: Enabled + statusController: + maxConcurrentReconciles: 1 `) func configKubefedconfigYamlBytes() ([]byte, error) { From 1cc5d6a1c0fb2edfc4ef92a78c560681705ff510 Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Tue, 4 May 2021 19:41:35 +0800 Subject: [PATCH 5/6] chore: add controller concurrency settings to chart --- charts/kubefed/README.md | 4 +++- .../kubefed/charts/controllermanager/crds/crds.yaml | 13 +++++++++++++ .../controllermanager/templates/kubefedconfig.yaml | 3 +++ charts/kubefed/values.yaml | 3 +++ 4 files changed, 22 insertions(+), 1 deletion(-) diff --git a/charts/kubefed/README.md b/charts/kubefed/README.md index 237dc096c9..7e771bf565 100644 --- a/charts/kubefed/README.md +++ b/charts/kubefed/README.md @@ -123,7 +123,9 @@ chart and their default values. | controllermanager.clusterHealthCheckFailureThreshold | Minimum consecutive failures for the cluster health to be considered failed after having succeeded. | 3 | | controllermanager.clusterHealthCheckSuccessThreshold | Minimum consecutive successes for the cluster health to be considered successful after having failed. | 1 | | controllermanager.clusterHealthCheckTimeout | Duration after which the cluster health check times out. | 3s | -| controllermanager.syncController.adoptResources | Whether to adopt pre-existing resource in member clusters. | Enabled | +| controllermanager.syncController.maxConcurrentReconciles | The maximum number of concurrent Reconciles of sync controller which can be run. | 1 | +| controllermanager.syncController.adoptResources | Whether to adopt pre-existing resource in member clusters. | Enabled | +| controllermanager.statusController.maxConcurrentReconciles | The maximum number of concurrent Reconciles of status controller which can be run. | 1 | | controllermanager.service.labels | Kubernetes labels attached to the controller manager's services | {} | | controllermanager.certManager.enabled | Specifies whether to enable the usage of the cert-manager for the certificates generation. | false | | controllermanager.certManager.rootCertificate.organizations | Specifies the list of organizations to include in the cert-manager generated root certificate. | [] | diff --git a/charts/kubefed/charts/controllermanager/crds/crds.yaml b/charts/kubefed/charts/controllermanager/crds/crds.yaml index 98858fdd03..30ff31d9c9 100644 --- a/charts/kubefed/charts/controllermanager/crds/crds.yaml +++ b/charts/kubefed/charts/controllermanager/crds/crds.yaml @@ -692,12 +692,25 @@ spec: `Namespaced` or `Cluster`. `Namespaced` indicates that the KubeFed namespace will be the only target of the control plane. type: string + statusController: + properties: + maxConcurrentReconciles: + description: The maximum number of concurrent Reconciles of status + controller which can be run. Defaults to 1. + format: int64 + type: integer + type: object syncController: properties: adoptResources: description: Whether to adopt pre-existing resources in member clusters. Defaults to "Enabled". type: string + maxConcurrentReconciles: + description: The maximum number of concurrent Reconciles of sync + controller which can be run. Defaults to 1. + format: int64 + type: integer type: object required: - scope diff --git a/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml b/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml index 34110b38cf..f0c3071613 100644 --- a/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml +++ b/charts/kubefed/charts/controllermanager/templates/kubefedconfig.yaml @@ -19,7 +19,10 @@ spec: successThreshold: {{ .Values.clusterHealthCheckSuccessThreshold | default 1 }} timeout: {{ .Values.clusterHealthCheckTimeout | default "3s" | quote }} syncController: + maxConcurrentReconciles: {{ .Values.syncController.maxConcurrentReconciles | default 1 }} adoptResources: {{ .Values.syncController.adoptResources | default "Enabled" | quote }} + statusController: + maxConcurrentReconciles: {{ .Values.statusController.maxConcurrentReconciles | default 1 }} featureGates: {{- if .Values.featureGates }} - name: PushReconciler diff --git a/charts/kubefed/values.yaml b/charts/kubefed/values.yaml index 813c3ec238..c37715cfd3 100644 --- a/charts/kubefed/values.yaml +++ b/charts/kubefed/values.yaml @@ -18,7 +18,10 @@ controllermanager: ## Supported options are `configmaps` and `endpoints` leaderElectResourceLock: syncController: + maxConcurrentReconciles: adoptResources: + statusController: + maxConcurrentReconciles: ## Value of feature gates item should be either `Enabled` or `Disabled` featureGates: PushReconciler: From 661df5d887f8ce74f2ecec2c34ab3a24a6b08e9b Mon Sep 17 00:00:00 2001 From: Zach Zhu Date: Tue, 4 May 2021 19:43:47 +0800 Subject: [PATCH 6/6] fix typo in defaults test --- pkg/apis/core/v1beta1/defaults/defaults_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/core/v1beta1/defaults/defaults_test.go b/pkg/apis/core/v1beta1/defaults/defaults_test.go index 2975917b3d..e093be7bdf 100644 --- a/pkg/apis/core/v1beta1/defaults/defaults_test.go +++ b/pkg/apis/core/v1beta1/defaults/defaults_test.go @@ -131,7 +131,7 @@ func TestSetDefaultKubeFedConfig(t *testing.T) { *adoptResourcesKFC.Spec.SyncController.AdoptResources = v1beta1.AdoptResourcesDisabled modifiedAdoptResourcesKFC := adoptResourcesKFC.DeepCopyObject().(*v1beta1.KubeFedConfig) SetDefaultKubeFedConfig(modifiedAdoptResourcesKFC) - successCases["spec.leaderElect.adoptResources is preserved"] = KubeFedConfigComparison{adoptResourcesKFC, modifiedAdoptResourcesKFC} + successCases["spec.syncController.adoptResources is preserved"] = KubeFedConfigComparison{adoptResourcesKFC, modifiedAdoptResourcesKFC} // StatusController statusControllerMaxConcurrentReconcilesKFC := defaultKubeFedConfig()