Merge #117519

117519: roachtest: fail the latency verifier immediately if latency is too high r=stevendanna a=msbutler

Previously, the latency verifier used by c2c and cdc roachtests would not fail immediately if latency was too high. This patch fails the latency verifier immediately, as the roachtest is going to eventually fail anyway.

In addition, immediately after a latency verifier failure, this patch captures c2c debug zips to better understand the cause of the increased latency.

Fixes #117182

Release note: none

Co-authored-by: Michael Butler <[email protected]>

pkg/cmd/roachtest/tests/cluster_to_cluster.go

@@ -930,7 +930,14 @@ func (rd *replicationDriver) main(ctx context.Context) {
 
 	latencyMonitor := rd.newMonitor(ctx)
 	latencyMonitor.Go(func(ctx context.Context) error {
-		return lv.pollLatencyUntilJobSucceeds(ctx, rd.setup.dst.db, ingestionJobID, time.Second, workloadDoneCh)
+		if err := lv.pollLatencyUntilJobSucceeds(ctx, rd.setup.dst.db, ingestionJobID, time.Second, workloadDoneCh); err != nil {
+			// The latency poller may have failed because latency got too high. Grab a
+			// debug zip before the replication jobs spin down.
+			rd.fetchDebugZip(ctx, rd.setup.src.nodes, "latency_source_debug.zip")
+			rd.fetchDebugZip(ctx, rd.setup.dst.nodes, "latency_dest_debug.zip")
+			return err
+		}
+		return nil
 	})
 	defer latencyMonitor.Wait()
 

pkg/cmd/roachtest/tests/latency_verifier.go

@@ -181,6 +181,10 @@ func (lv *latencyVerifier) pollLatencyUntilJobSucceeds(
 			lv.logger.Printf("unexpected status: %s, error: %s", status, info.GetError())
 			return errors.Errorf("unexpected status: %s", status)
 		}
+		if lv.targetSteadyLatency != 0 && lv.maxSeenSteadyLatency > lv.targetSteadyLatency {
+			return errors.Errorf("max latency was more than allowed: %s vs %s",
+				lv.maxSeenSteadyLatency, lv.targetSteadyLatency)
+		}
 	}
 }