admission: Split stats for WorkQueueMetrics

This commit splits the WorkQueueMetric stats into priority.
Because there are a lot of new stats that might be generated
as a result of this splitting, only the "important" priorities
are collected and considered for each queue.
Informs cockroachdb#82743.

Release note: None

pkg/kv/kvserver/client_split_test.go

@@ -1221,7 +1221,7 @@ func TestStoreRangeSplitBackpressureWrites(t *testing.T) {
 
 			// Set maxBytes to something small so we can exceed the maximum split
 			// size without adding 2x64MB of data.
-			const maxBytes = 1 << 16
+			const maxBytes = 1 << 17
 			zoneConfig := zonepb.DefaultZoneConfig()
 			zoneConfig.RangeMaxBytes = proto.Int64(maxBytes)
 
@@ -2539,7 +2539,7 @@ func TestUnsplittableRange(t *testing.T) {
 
 	ctx := context.Background()
 	ttl := 1 * time.Hour
-	const maxBytes = 1 << 16
+	const maxBytes = 1 << 17
 	manualClock := hlc.NewHybridManualClock()
 	zoneConfig := zonepb.DefaultZoneConfig()
 	zoneConfig.RangeMaxBytes = proto.Int64(maxBytes)

pkg/server/server.go

@@ -219,8 +219,7 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
 	if opts, ok := cfg.TestingKnobs.AdmissionControl.(*admission.Options); ok {
 		admissionOptions.Override(opts)
 	}
-	admissionOptions.Settings = st
-	gcoords, metrics := admission.NewGrantCoordinators(cfg.AmbientCtx, admissionOptions)
+	gcoords := admission.NewGrantCoordinators(cfg.AmbientCtx, st, admissionOptions, registry)
 	ssg, err := admission.MakeSoftSlotGranter(gcoords.Regular)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to soft slot granter")
@@ -392,9 +391,6 @@ func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
 	}
 	tcsFactory := kvcoord.NewTxnCoordSenderFactory(txnCoordSenderFactoryCfg, distSender)
 
-	for i := range metrics {
-		registry.AddMetricStruct(metrics[i])
-	}
 	cbID := goschedstats.RegisterRunnableCountCallback(gcoords.Regular.CPULoad)
 	stopper.AddCloser(stop.CloserFn(func() {
 		goschedstats.UnregisterRunnableCountCallback(cbID)

pkg/ts/catalog/chart_catalog.go

@@ -3460,6 +3460,51 @@ var charts = []sectionDescription{
 					"admission.requested.elastic-cpu",
 					"admission.admitted.elastic-cpu",
 					"admission.errored.elastic-cpu",
+					"admission.admitted.elastic-cpu.bulk",
+					"admission.admitted.kv-stores.bulk",
+					"admission.admitted.kv-stores.locking",
+					"admission.admitted.kv-stores.ttl",
+					"admission.admitted.kv-stores.normal",
+					"admission.admitted.kv.locking",
+					"admission.admitted.kv.normal",
+					"admission.admitted.sql-kv-response.locking",
+					"admission.admitted.sql-kv-response.normal",
+					"admission.admitted.sql-leaf-start.locking",
+					"admission.admitted.sql-leaf-start.normal",
+					"admission.admitted.sql-root-start.locking",
+					"admission.admitted.sql-root-start.normal",
+					"admission.admitted.sql-sql-response.locking",
+					"admission.admitted.sql-sql-response.normal",
+					"admission.errored.elastic-cpu.bulk",
+					"admission.errored.kv-stores.bulk",
+					"admission.errored.kv-stores.locking",
+					"admission.errored.kv-stores.ttl",
+					"admission.errored.kv-stores.normal",
+					"admission.errored.kv.locking",
+					"admission.errored.kv.normal",
+					"admission.errored.sql-kv-response.locking",
+					"admission.errored.sql-kv-response.normal",
+					"admission.errored.sql-leaf-start.locking",
+					"admission.errored.sql-leaf-start.normal",
+					"admission.errored.sql-root-start.locking",
+					"admission.errored.sql-root-start.normal",
+					"admission.errored.sql-sql-response.locking",
+					"admission.errored.sql-sql-response.normal",
+					"admission.requested.elastic-cpu.bulk",
+					"admission.requested.kv-stores.bulk",
+					"admission.requested.kv-stores.locking",
+					"admission.requested.kv-stores.ttl",
+					"admission.requested.kv-stores.normal",
+					"admission.requested.kv.locking",
+					"admission.requested.kv.normal",
+					"admission.requested.sql-kv-response.locking",
+					"admission.requested.sql-kv-response.normal",
+					"admission.requested.sql-leaf-start.locking",
+					"admission.requested.sql-leaf-start.normal",
+					"admission.requested.sql-root-start.locking",
+					"admission.requested.sql-root-start.normal",
+					"admission.requested.sql-sql-response.locking",
+					"admission.requested.sql-sql-response.normal",
 				},
 			},
 			{
@@ -3472,18 +3517,21 @@ var charts = []sectionDescription{
 					"admission.wait_queue_length.sql-leaf-start",
 					"admission.wait_queue_length.sql-root-start",
 					"admission.wait_queue_length.elastic-cpu",
-				},
-			},
-			{
-				Title: "Work Queue Admission Latency Sum",
-				Metrics: []string{
-					"admission.wait_sum.kv",
-					"admission.wait_sum.kv-stores",
-					"admission.wait_sum.sql-kv-response",
-					"admission.wait_sum.sql-sql-response",
-					"admission.wait_sum.sql-leaf-start",
-					"admission.wait_sum.sql-root-start",
-					"admission.wait_sum.elastic-cpu",
+					"admission.wait_queue_length.elastic-cpu.bulk",
+					"admission.wait_queue_length.kv-stores.bulk",
+					"admission.wait_queue_length.kv-stores.locking",
+					"admission.wait_queue_length.kv-stores.ttl",
+					"admission.wait_queue_length.kv-stores.normal",
+					"admission.wait_queue_length.kv.locking",
+					"admission.wait_queue_length.kv.normal",
+					"admission.wait_queue_length.sql-kv-response.locking",
+					"admission.wait_queue_length.sql-kv-response.normal",
+					"admission.wait_queue_length.sql-leaf-start.locking",
+					"admission.wait_queue_length.sql-leaf-start.normal",
+					"admission.wait_queue_length.sql-root-start.locking",
+					"admission.wait_queue_length.sql-root-start.normal",
+					"admission.wait_queue_length.sql-sql-response.locking",
+					"admission.wait_queue_length.sql-sql-response.normal",
 				},
 			},
 			{
@@ -3496,6 +3544,21 @@ var charts = []sectionDescription{
 					"admission.wait_durations.sql-leaf-start",
 					"admission.wait_durations.sql-root-start",
 					"admission.wait_durations.elastic-cpu",
+					"admission.wait_durations.elastic-cpu.bulk",
+					"admission.wait_durations.kv-stores.bulk",
+					"admission.wait_durations.kv-stores.locking",
+					"admission.wait_durations.kv-stores.ttl",
+					"admission.wait_durations.kv-stores.normal",
+					"admission.wait_durations.kv.locking",
+					"admission.wait_durations.kv.normal",
+					"admission.wait_durations.sql-kv-response.locking",
+					"admission.wait_durations.sql-kv-response.normal",
+					"admission.wait_durations.sql-leaf-start.locking",
+					"admission.wait_durations.sql-leaf-start.normal",
+					"admission.wait_durations.sql-root-start.locking",
+					"admission.wait_durations.sql-root-start.normal",
+					"admission.wait_durations.sql-sql-response.locking",
+					"admission.wait_durations.sql-sql-response.normal",
 				},
 			},
 			{

pkg/util/admission/BUILD.bazel

@@ -67,6 +67,7 @@ go_test(
         "//pkg/util/admission/admissionpb",
         "//pkg/util/leaktest",
         "//pkg/util/log",
+        "//pkg/util/metric",
         "//pkg/util/quotapool",
         "//pkg/util/syncutil",
         "//pkg/util/timeutil",

pkg/util/admission/admission.go

@@ -133,7 +133,6 @@ import (
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble"
-	"github.com/cockroachdb/redact"
 )
 
 // requester is an interface implemented by an object that orders admission
@@ -500,7 +499,7 @@ const (
 	numWorkKinds
 )
 
-func workKindString(workKind WorkKind) redact.RedactableString {
+func workKindString(workKind WorkKind) string {
 	switch workKind {
 	case KVWork:
 		return "kv"

pkg/util/admission/admissionpb/admissionpb.go

@@ -39,6 +39,17 @@ const (
 	OneAboveHighPri int = int(HighPri) + 1
 )
 
+var WorkPriorityDict = map[WorkPriority]string{
+	LowPri:        "min",
+	TTLLowPri:     "ttl",
+	UserLowPri:    "low",
+	BulkNormalPri: "bulk",
+	NormalPri:     "normal",
+	UserHighPri:   "high",
+	LockingPri:    "locking",
+	HighPri:       "max",
+}
+
 // Prevent the linter from emitting unused warnings.
 var _ = LowPri
 var _ = TTLLowPri