From 2afc9395884cfd746f0b1fe1cadadfb96d9b8c82 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 22 Nov 2019 18:41:21 -0500 Subject: [PATCH] tests: deflake TestHTTP_FreshClientAllocMetrics The test asserts that alloc counts get reported accurately in metrics by inspecting the metrics endpoint directly. Sadly, the metrics as collected by `armon/go-metrics` seem to be stateful and may contain info from other tests. This means that the test can fail depending on the order of returned metrics. Inspecting the metrics output of one failing run, you can see the duplicate guage entries but for different node_ids: ``` { "Name": "service-name.default-0a3ba4b6-2109-485e-be74-6864228aed3d.client.allocations.terminal", "Value": 10, "Labels": { "datacenter": "dc1", "node_class": "none", "node_id": "67402bf4-00f3-bd8d-9fa8-f4d1924a892a" } }, { "Name": "service-name.default-0a3ba4b6-2109-485e-be74-6864228aed3d.client.allocations.terminal", "Value": 0, "Labels": { "datacenter": "dc1", "node_class": "none", "node_id": "a2945b48-7e66-68e2-c922-49b20dd4e20c" } }, ``` --- command/agent/metrics_endpoint_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/command/agent/metrics_endpoint_test.go b/command/agent/metrics_endpoint_test.go index d97fe2e69ca..b10e4170865 100644 --- a/command/agent/metrics_endpoint_test.go +++ b/command/agent/metrics_endpoint_test.go @@ -91,6 +91,8 @@ func TestHTTP_FreshClientAllocMetrics(t *testing.T) { require.Fail("timed-out waiting for job to complete") }) + nodeID := s.client.NodeID() + // wait for metrics to converge var pending, running, terminal float32 = -1.0, -1.0, -1.0 testutil.WaitForResultRetries(100, func() (bool, error) { @@ -106,6 +108,13 @@ func TestHTTP_FreshClientAllocMetrics(t *testing.T) { metrics := obj.(metrics.MetricsSummary) for _, g := range metrics.Gauges { + + // ignore client metrics belonging to other test nodes + // from other tests that contaminate go-metrics reporting + if g.DisplayLabels["node_id"] != nodeID { + continue + } + if strings.HasSuffix(g.Name, "client.allocations.pending") { pending = g.Value }