From aa11fbf56848d8fefe5d7c0bfde05f146c070f75 Mon Sep 17 00:00:00 2001 From: Andrew Baptist Date: Thu, 20 Oct 2022 22:28:36 -0400 Subject: [PATCH] admission: temporary knob to disable AC for user requests informs: #85641 Temporary knob to allow disabling admission control for non-elastic / bulk requests. Release note: None --- pkg/kv/kvserver/store.go | 6 ++++++ pkg/util/admission/work_queue.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/pkg/kv/kvserver/store.go b/pkg/kv/kvserver/store.go index 56fde35df682..4d393e0773f0 100644 --- a/pkg/kv/kvserver/store.go +++ b/pkg/kv/kvserver/store.go @@ -3728,6 +3728,12 @@ func (n KVAdmissionControllerImpl) AdmitKVWork( if source == roachpb.AdmissionHeader_OTHER { bypassAdmission = true } + // TODO(abaptist): Revisit and deprecate this setting in v23.1. + if admission.KVBulkOnlyAdmissionControlEnabled.Get(&n.settings.SV) { + if admission.WorkPriority(ba.AdmissionHeader.Priority) >= admission.NormalPri { + bypassAdmission = true + } + } createTime := ba.AdmissionHeader.CreateTime if !bypassAdmission && createTime == 0 { // TODO(sumeer): revisit this for multi-tenant. Specifically, the SQL use diff --git a/pkg/util/admission/work_queue.go b/pkg/util/admission/work_queue.go index fc86f6fa451f..0c906e88c83e 100644 --- a/pkg/util/admission/work_queue.go +++ b/pkg/util/admission/work_queue.go @@ -49,6 +49,20 @@ var KVAdmissionControlEnabled = settings.RegisterBoolSetting( "when true, work performed by the KV layer is subject to admission control", true).WithPublic() +// KVBulkOnlyAdmissionControlEnabled controls whether user (normal and above +// priority) work is subject to admission control. If it is set to true, then +// user work will not be throttled by admission control but bulk work still will +// be. This setting is a preferable alternative to completely disabling +// admission control. It can be used reactively in cases where index backfill, +// schema modifications or other bulk operations are causing high latency due to +// io_overload on nodes. +// TODO(baptist): Find a better solution to this in v23.1. +var KVBulkOnlyAdmissionControlEnabled = settings.RegisterBoolSetting( + settings.SystemOnly, + "admission.kv.bulk_only.enabled", + "when both admission.kv.enabled and this is true, only throttle bulk work", + false) + // SQLKVResponseAdmissionControlEnabled controls whether response processing // in SQL, for KV requests, is enabled. var SQLKVResponseAdmissionControlEnabled = settings.RegisterBoolSetting(