From 52a0e006c4fd31bc0c71d8d876b9e6528b39c8ae Mon Sep 17 00:00:00 2001
From: Aayush Shah <aayush.shah15@gmail.com>
Date: Mon, 7 Nov 2022 15:51:19 -0500
Subject: [PATCH] kvserver: add metrics for when read-only batches drop latches

Release note: None
---
 pkg/kv/kvserver/metrics.go      | 19 ++++++++++++++++++-
 pkg/kv/kvserver/replica_read.go |  2 ++
 pkg/ts/catalog/chart_catalog.go | 12 ++++++++++++
 3 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/pkg/kv/kvserver/metrics.go b/pkg/kv/kvserver/metrics.go
index dd1498d5c424..368fb07989a9 100644
--- a/pkg/kv/kvserver/metrics.go
+++ b/pkg/kv/kvserver/metrics.go
@@ -1651,7 +1651,6 @@ Note that the measurement does not include the duration for replicating the eval
 		Measurement: "Nanoseconds",
 		Unit:        metric.Unit_NANOSECONDS,
 	}
-
 	metaPopularKeyCount = metric.Metadata{
 		Name:        "kv.loadsplitter.popularkey",
 		Help:        "Load-based splitter could not find a split key and the most popular sampled split key occurs in >= 25% of the samples.",
@@ -1679,6 +1678,18 @@ Note that the measurement does not include the duration for replicating the eval
 		Measurement: "Fsync Latency",
 		Unit:        metric.Unit_NANOSECONDS,
 	}
+	metaReplicaReadBatchDroppedLatchesBeforeEval = metric.Metadata{
+		Name:        "kv.replica_read_batch_evaluate.dropped_latches_before_eval",
+		Help:        `Number of times read-only batches dropped latches before evaluation.`,
+		Measurement: "Batches",
+		Unit:        metric.Unit_COUNT,
+	}
+	metaReplicaReadBatchWithoutInterleavingIter = metric.Metadata{
+		Name:        "kv.replica_read_batch_evaluate.without_interleaving_iter",
+		Help:        `Number of read-only batches evaluated without an intent interleaving iter.`,
+		Measurement: "Batches",
+		Unit:        metric.Unit_COUNT,
+	}
 )
 
 // StoreMetrics is the set of metrics for a given store.
@@ -1975,6 +1986,9 @@ type StoreMetrics struct {
 	ReplicaReadBatchEvaluationLatency  *metric.Histogram
 	ReplicaWriteBatchEvaluationLatency *metric.Histogram
 
+	ReplicaReadBatchDroppedLatchesBeforeEval *metric.Counter
+	ReplicaReadBatchWithoutInterleavingIter  *metric.Counter
+
 	FlushUtilization *metric.GaugeFloat64
 	FsyncLatency     *metric.ManualWindowHistogram
 }
@@ -2517,6 +2531,9 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		),
 		FlushUtilization: metric.NewGaugeFloat64(metaStorageFlushUtilization),
 		FsyncLatency:     metric.NewManualWindowHistogram(metaStorageFsyncLatency, pebble.FsyncLatencyBuckets),
+
+		ReplicaReadBatchDroppedLatchesBeforeEval: metric.NewCounter(metaReplicaReadBatchDroppedLatchesBeforeEval),
+		ReplicaReadBatchWithoutInterleavingIter:  metric.NewCounter(metaReplicaReadBatchWithoutInterleavingIter),
 	}
 
 	{
diff --git a/pkg/kv/kvserver/replica_read.go b/pkg/kv/kvserver/replica_read.go
index 2e9b3012445a..de7e521ba75f 100644
--- a/pkg/kv/kvserver/replica_read.go
+++ b/pkg/kv/kvserver/replica_read.go
@@ -109,7 +109,9 @@ func (r *Replica) executeReadOnlyBatch(
 		// the request is not being optimistically evaluated (optimistic evaluation
 		// does not wait for latches or check locks).
 		log.VEventf(ctx, 3, "lock table scan complete without conflicts; dropping latches early")
+		r.store.metrics.ReplicaReadBatchDroppedLatchesBeforeEval.Inc(1)
 		if !stillNeedsInterleavedIntents {
+			r.store.metrics.ReplicaReadBatchWithoutInterleavingIter.Inc(1)
 			evalPath = readOnlyWithoutInterleavedIntents
 		}
 		r.updateTimestampCacheAndDropLatches(ctx, g, ba, nil /* br */, nil /* pErr */, st)
diff --git a/pkg/ts/catalog/chart_catalog.go b/pkg/ts/catalog/chart_catalog.go
index 0ca5853957e5..5368c0788b26 100644
--- a/pkg/ts/catalog/chart_catalog.go
+++ b/pkg/ts/catalog/chart_catalog.go
@@ -3750,4 +3750,16 @@ var charts = []sectionDescription{
 			},
 		},
 	},
+	{
+		Organization: [][]string{{ReplicationLayer, "Batches"}},
+		Charts: []chartDescription{
+			{
+				Title: "Total number of attempts to evaluate read-only batches",
+				Metrics: []string{
+					"kv.replica_read_batch_evaluate.dropped_latches_before_eval",
+					"kv.replica_read_batch_evaluate.without_interleaving_iter",
+				},
+			},
+		},
+	},
 }