From 5b6e61b993e8fbd9a30d889afbce9df5ea73235e Mon Sep 17 00:00:00 2001
From: Nick Ripley <97066770+nsrip-dd@users.noreply.github.com>
Date: Thu, 27 Jan 2022 08:09:29 -0500
Subject: [PATCH] internal/traceprof/traceproftest: fix flaxy
 endpoints/hotspots test (#1145)

* internal/traceprof/traceprof: fix flaxy endpoints/hotspots test

TestEndpointsAndCodeHotspots fails occasionally in CI. Change the retry
logic so that every required label is checked for in testCommon before
ending the retry loop.

* internal/traceprof/traceproftest: identify reason for test failure

If TestEndpointsAndCodeHotspots fails after too many attempts, show
which condition wasn't met after all the attempts.
---
 .../traceprof/traceproftest/traceprof_test.go | 28 +++++++++++++------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/internal/traceprof/traceproftest/traceprof_test.go b/internal/traceprof/traceproftest/traceprof_test.go
index e3353ec1b3..45d5a19168 100644
--- a/internal/traceprof/traceproftest/traceprof_test.go
+++ b/internal/traceprof/traceproftest/traceprof_test.go
@@ -42,18 +42,22 @@ func TestEndpointsAndCodeHotspots(t *testing.T) {
 
 		// Rerun test a few times with doubled duration until it passes to avoid
 		// flaky behavior in CI.
-		for attempt := 1; ; attempt++ {
+		var (
+			prof *CPUProfile
+			res  *pb.WorkRes
+		)
+		for attempt := 1; attempt <= 5; attempt++ {
 			app := c.Start(t)
 			defer app.Stop(t)
 
-			res := app.WorkRequest(t, req)
-			prof := app.CPUProfile(t)
+			res = app.WorkRequest(t, req)
+			prof = app.CPUProfile(t)
 
 			notEnoughSamples := (prof.Duration() < minCPUDuration) ||
 				(prof.LabelsDuration(CustomLabels) < minCPUDuration) ||
-				(c.CodeHotspots && prof.LabelDuration(traceprof.SpanID, "*") < minCPUDuration) ||
-				(c.AppType != Direct && c.Endpoints && prof.LabelDuration(traceprof.TraceEndpoint, "*") < minCPUDuration)
-			if attempt <= 5 && notEnoughSamples {
+				(c.CodeHotspots && prof.LabelsDuration(map[string]string{traceprof.LocalRootSpanID: res.LocalRootSpanId, traceprof.SpanID: res.SpanId}) < minCPUDuration) ||
+				(c.Endpoints && prof.LabelDuration(traceprof.TraceEndpoint, c.AppType.Endpoint()) < minCPUDuration)
+			if notEnoughSamples {
 				req.CpuDuration *= 2
 				req.SqlDuration *= 2
 				t.Logf("attempt %d: not enough cpu samples, doubling duration", attempt)
@@ -61,10 +65,18 @@ func TestEndpointsAndCodeHotspots(t *testing.T) {
 			}
 			require.True(t, ValidSpanID(res.SpanId))
 			require.True(t, ValidSpanID(res.LocalRootSpanId))
-			require.GreaterOrEqual(t, prof.Duration(), minCPUDuration)
-			require.GreaterOrEqual(t, prof.LabelsDuration(CustomLabels), minCPUDuration)
 			return res, prof
 		}
+		// Failed after 5 attempts, identify which condition wasn't met
+		require.GreaterOrEqual(t, prof.Duration(), minCPUDuration)
+		require.GreaterOrEqual(t, prof.LabelsDuration(CustomLabels), minCPUDuration)
+		if c.Endpoints {
+			require.GreaterOrEqual(t, prof.LabelDuration(traceprof.TraceEndpoint, c.AppType.Endpoint()), minCPUDuration)
+		}
+		if c.CodeHotspots {
+			require.GreaterOrEqual(t, prof.LabelsDuration(map[string]string{traceprof.LocalRootSpanID: res.LocalRootSpanId, traceprof.SpanID: res.SpanId}), minCPUDuration)
+		}
+		return nil, nil
 	}
 
 	for _, appType := range []testAppType{Direct, HTTP, GRPC} {