Merge cockroachdb#37027

37027: roachtest: deflake replicate/wide r=tbg a=petermattis Only fatal on non-zero unavailable ranges if that condition persists for more than 30s. Fixes cockroachdb#36788 Release note: None Co-authored-by: Peter Mattis <[email protected]>
bdarnell · Apr 23, 2019 · 1841bc4 · 1841bc4
2 parents 7424c53 + 04d6011
commit 1841bc4
Showing 1 changed file with 8 additions and 2 deletions.
diff --git a/pkg/cmd/roachtest/allocator.go b/pkg/cmd/roachtest/allocator.go
@@ -339,7 +339,7 @@ func runWideReplication(ctx context.Context, t *test, c *cluster) {
 	c.Start(ctx, t, c.Range(1, 6), args)
 
 	waitForUnderReplicated := func(count int) {
-		for ; ; time.Sleep(time.Second) {
+		for start := timeutil.Now(); ; time.Sleep(time.Second) {
 			query := `
 SELECT sum((metrics->>'ranges.unavailable')::DECIMAL)::INT AS ranges_unavailable,
        sum((metrics->>'ranges.underreplicated')::DECIMAL)::INT AS ranges_underreplicated
@@ -351,7 +351,13 @@ FROM crdb_internal.kv_store_status
 			}
 			t.l.Printf("%d unavailable, %d under-replicated ranges\n", unavailable, underReplicated)
 			if unavailable != 0 {
-				t.Fatalf("%d unavailable ranges", unavailable)
+				// A freshly started cluster might show unavailable ranges for a brief
+				// period of time due to the way that metric is calculated. Only
+				// complain about unavailable ranges if they persist for too long.
+				if timeutil.Since(start) >= 30*time.Second {
+					t.Fatalf("%d unavailable ranges", unavailable)
+				}
+				continue
 			}
 			if underReplicated >= count {
 				break