From f456c2271f1461369c5371828990085f1dcef34a Mon Sep 17 00:00:00 2001
From: Aaditya Sondhi <20070511+aadityasondhi@users.noreply.github.com>
Date: Fri, 17 Nov 2023 14:12:29 -0500
Subject: [PATCH] admission: include work priority in trace and errors

This patch now includes work priority in the `WorkQueue` traces and
error messages to better help track where work items originate from.

Informs #113990.

Release note: None
---
 .../admission/admissionpb/admission_stats.proto |  3 ++-
 pkg/util/admission/work_queue.go                | 17 +++++++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/pkg/util/admission/admissionpb/admission_stats.proto b/pkg/util/admission/admissionpb/admission_stats.proto
index eee3d82fd63b..3f48df2de1a4 100644
--- a/pkg/util/admission/admissionpb/admission_stats.proto
+++ b/pkg/util/admission/admissionpb/admission_stats.proto
@@ -23,5 +23,6 @@ message AdmissionWorkQueueStats {
   string queue_kind = 2;
   // Set to true if deadline was exceeded.
   bool deadline_exceeded = 3;
-
+  // String representation of work priority.
+  string work_priority = 4;
 }
diff --git a/pkg/util/admission/work_queue.go b/pkg/util/admission/work_queue.go
index 4f0939a47d4c..81c0facb0d59 100644
--- a/pkg/util/admission/work_queue.go
+++ b/pkg/util/admission/work_queue.go
@@ -814,11 +814,11 @@ func (q *WorkQueue) Admit(ctx context.Context, info WorkInfo) (enabled bool, err
 		q.metrics.incErrored(info.Priority)
 		q.metrics.recordFinishWait(info.Priority, waitDur)
 		deadline, _ := ctx.Deadline()
-		recordAdmissionWorkQueueStats(span, waitDur, q.queueKind, true)
-		log.Eventf(ctx, "deadline expired, waited in %s queue for %v", q.queueKind, waitDur)
+		recordAdmissionWorkQueueStats(span, waitDur, q.queueKind, info.Priority, true)
+		log.Eventf(ctx, "deadline expired, waited in %s queue with pri %s for %v", q.queueKind, admissionpb.WorkPriorityDict[info.Priority], waitDur)
 		return true,
-			errors.Newf("deadline expired while waiting in queue: %s, deadline: %v, start: %v, dur: %v",
-				q.queueKind, deadline, startTime, waitDur)
+			errors.Newf("deadline expired while waiting in queue: %s, pri: %s, deadline: %v, start: %v, dur: %v",
+				q.queueKind, admissionpb.WorkPriorityDict[info.Priority], deadline, startTime, waitDur)
 	case chainID, ok := <-work.ch:
 		if !ok {
 			panic(errors.AssertionFailedf("channel should not be closed"))
@@ -829,14 +829,18 @@ func (q *WorkQueue) Admit(ctx context.Context, info WorkInfo) (enabled bool, err
 		if work.heapIndex != -1 {
 			panic(errors.AssertionFailedf("grantee should be removed from heap"))
 		}
-		recordAdmissionWorkQueueStats(span, waitDur, q.queueKind, false)
+		recordAdmissionWorkQueueStats(span, waitDur, q.queueKind, info.Priority, false)
 		q.granter.continueGrantChain(chainID)
 		return true, nil
 	}
 }
 
 func recordAdmissionWorkQueueStats(
-	span *tracing.Span, waitDur time.Duration, queueKind QueueKind, deadlineExceeded bool,
+	span *tracing.Span,
+	waitDur time.Duration,
+	queueKind QueueKind,
+	workPriority admissionpb.WorkPriority,
+	deadlineExceeded bool,
 ) {
 	if span == nil {
 		return
@@ -845,6 +849,7 @@ func recordAdmissionWorkQueueStats(
 		WaitDurationNanos: waitDur,
 		QueueKind:         string(queueKind),
 		DeadlineExceeded:  deadlineExceeded,
+		WorkPriority:      admissionpb.WorkPriorityDict[workPriority],
 	})
 }