admission: temporary knob to disable AC for user requests

informs: cockroachdb#85641

Temporary knob to allow disabling admission control for non-elastic /
bulk requests.

Release note: None

pkg/kv/kvserver/store.go

@@ -3910,6 +3910,13 @@ func (n *KVAdmissionControllerImpl) AdmitKVWork(
 	if source == roachpb.AdmissionHeader_OTHER {
 		bypassAdmission = true
 	}
+	// TODO(abaptist): Revisit and deprecate this setting in v23.1.
+	if admission.KVBulkOnlyAdmissionControlEnabled.Get(&n.settings.SV) {
+		if admissionpb.WorkPriority(ba.AdmissionHeader.Priority) >= admissionpb.NormalPri {
+			bypassAdmission = true
+		}
+	}
+
 	createTime := ba.AdmissionHeader.CreateTime
 	if !bypassAdmission && createTime == 0 {
 		// TODO(sumeer): revisit this for multi-tenant. Specifically, the SQL use

pkg/util/admission/work_queue.go

@@ -51,6 +51,20 @@ var KVAdmissionControlEnabled = settings.RegisterBoolSetting(
 	"when true, work performed by the KV layer is subject to admission control",
 	true).WithPublic()
 
+// KVBulkOnlyAdmissionControlEnabled controls whether user (normal and above
+// priority) work is subject to admission control. If it is set to true, then
+// user work will not be throttled by admission control but bulk work still will
+// be. This setting is a preferable alternative to completely disabling
+// admission control. It can be used reactively in cases where index backfill,
+// schema modifications or other bulk operations are causing high latency due to
+// io_overload on nodes.
+// TODO(baptist): Find a better solution to this in v23.1.
+var KVBulkOnlyAdmissionControlEnabled = settings.RegisterBoolSetting(
+	settings.SystemOnly,
+	"admission.kv.bulk_only.enabled",
+	"when both admission.kv.enabled and this is true, only throttle bulk work",
+	false)
+
 // SQLKVResponseAdmissionControlEnabled controls whether response processing
 // in SQL, for KV requests, is enabled.
 var SQLKVResponseAdmissionControlEnabled = settings.RegisterBoolSetting(