From 224e5e5a345583b8bb6a66dcf07af4dfc75b9680 Mon Sep 17 00:00:00 2001
From: Andrew Werner <ajwerner@cockroachlabs.com>
Date: Mon, 31 Aug 2020 08:59:07 -0400
Subject: [PATCH] kvserver/tenantrate: fix flakey test

This test was broken in #53510 which augmented the limiter machinery to
deal with read and write requests independently. The problem is that the
timer machinery works to synchronize the first blocked request but not
subsequent requests. That leaves the test-writer with no fundamental
means to ensure that subsequent blocked commands synchronize with a
metrics command. This lack of synchronization on metrics was a pain-point
when writing the original tests and led to excessive use of timers as
synchronization. Rather than trying to add excess synchronization, this
commit just adds a succeeds soon around the metrics. This works well.

Fixes #53590

Release justification: non-production code changes

Release note: None
---
 pkg/kv/kvserver/tenantrate/limiter_test.go | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/pkg/kv/kvserver/tenantrate/limiter_test.go b/pkg/kv/kvserver/tenantrate/limiter_test.go
index 2b46c9d3f225..1c94d70bb958 100644
--- a/pkg/kv/kvserver/tenantrate/limiter_test.go
+++ b/pkg/kv/kvserver/tenantrate/limiter_test.go
@@ -327,7 +327,8 @@ func (ts *testState) recordRead(t *testing.T, d *datadriven.TestData) string {
 
 // metrics will print out the prometheus metric values. The command takes an
 // argument as a regular expression over the values. The metrics are printed in
-// lexicographical order.
+// lexicographical order. The command will retry until the output matches to
+// make it more robust to races in metric recording.
 //
 // For example:
 //
@@ -358,6 +359,20 @@ func (ts *testState) recordRead(t *testing.T, d *datadriven.TestData) string {
 //  kv_tenant_rate_limit_write_bytes_admitted{tenant_id="2"} 50
 //
 func (ts *testState) metrics(t *testing.T, d *datadriven.TestData) string {
+	exp := strings.TrimSpace(d.Expected)
+	if err := testutils.SucceedsSoonError(func() error {
+		got := ts.getMetricsText(t, d)
+		if got != exp {
+			return errors.Errorf("got: %q, exp: %q", got, exp)
+		}
+		return nil
+	}); err != nil {
+		d.Fatalf(t, "failed to find expected timers: %v", err)
+	}
+	return d.Expected
+}
+
+func (ts *testState) getMetricsText(t *testing.T, d *datadriven.TestData) string {
 	ex := metric.MakePrometheusExporter()
 	ex.ScrapeRegistry(ts.m, true /* includeChildMetrics */)
 	var in bytes.Buffer
@@ -381,7 +396,8 @@ func (ts *testState) metrics(t *testing.T, d *datadriven.TestData) string {
 		d.Fatalf(t, "failed to process metrics: %v", err)
 	}
 	sort.Strings(outLines)
-	return strings.Join(outLines, "\n")
+	metricsText := strings.Join(outLines, "\n")
+	return metricsText
 }
 
 // timers waits for the set of open timers to match the expected output.