Add separate history for expired failed probe results (#517)

Signed-off-by: Jake Utley <[email protected]>

history.go

@@ -25,11 +25,15 @@ type result struct {
 	success     bool
 }
 
+// resultHistory contains two history slices: `results` contains most recent `maxResults` results.
+// After they expire out of `results`, failures will be saved in `preservedFailedResults`. This
+// ensures that we are always able to see debug information about recent failures.
 type resultHistory struct {
-	mu         sync.Mutex
-	nextId     int64
-	results    []*result
-	maxResults uint
+	mu                     sync.Mutex
+	nextId                 int64
+	results                []*result
+	preservedFailedResults []*result
+	maxResults             uint
 }
 
 // Add a result to the history.
@@ -48,6 +52,16 @@ func (rh *resultHistory) Add(moduleName, target, debugOutput string, success boo
 
 	rh.results = append(rh.results, r)
 	if uint(len(rh.results)) > rh.maxResults {
+		// If we are about to remove a failure, add it to the failed result history, then
+		// remove the oldest failed result, if needed.
+		if !rh.results[0].success {
+			rh.preservedFailedResults = append(rh.preservedFailedResults, rh.results[0])
+			if uint(len(rh.preservedFailedResults)) > rh.maxResults {
+				preservedFailedResults := make([]*result, len(rh.preservedFailedResults)-1)
+				copy(preservedFailedResults, rh.preservedFailedResults[1:])
+				rh.preservedFailedResults = preservedFailedResults
+			}
+		}
 		results := make([]*result, len(rh.results)-1)
 		copy(results, rh.results[1:])
 		rh.results = results
@@ -59,14 +73,20 @@ func (rh *resultHistory) List() []*result {
 	rh.mu.Lock()
 	defer rh.mu.Unlock()
 
-	return rh.results[:]
+	// Results in each slice are disjoint. We can simply concatenate the results.
+	return append(rh.preservedFailedResults[:], rh.results...)
 }
 
 // Get returns a given result.
 func (rh *resultHistory) Get(id int64) *result {
 	rh.mu.Lock()
 	defer rh.mu.Unlock()
 
+	for _, r := range rh.preservedFailedResults {
+		if r.id == id {
+			return r
+		}
+	}
 	for _, r := range rh.results {
 		if r.id == id {
 			return r

history_test.go

@@ -0,0 +1,81 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestHistoryKeepsLatestResults(t *testing.T) {
+	history := &resultHistory{maxResults: 3}
+	for i := 0; i < 4; i++ {
+		history.Add("module", "target", fmt.Sprintf("result %d", i), true)
+	}
+
+	savedResults := history.List()
+	for i := 0; i < len(savedResults); i++ {
+		if savedResults[i].debugOutput != fmt.Sprintf("result %d", i+1) {
+			t.Errorf("History contained the wrong result at index %d", i)
+		}
+	}
+}
+
+func FillHistoryWithMaxSuccesses(h *resultHistory) {
+	for i := uint(0); i < h.maxResults; i++ {
+		h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), true)
+	}
+}
+
+func FillHistoryWithMaxPreservedFailures(h *resultHistory) {
+	for i := uint(0); i < h.maxResults; i++ {
+		h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), false)
+	}
+}
+
+func TestHistoryPreservesExpiredFailedResults(t *testing.T) {
+	history := &resultHistory{maxResults: 3}
+
+	// Success are expired, no failues are expired
+	FillHistoryWithMaxSuccesses(history)
+	FillHistoryWithMaxPreservedFailures(history)
+	savedResults := history.List()
+	for i := uint(0); i < uint(len(savedResults)); i++ {
+		expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults)
+		if savedResults[i].debugOutput != expectedDebugOutput {
+			t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput)
+		}
+	}
+
+	// Failures are expired, should all be preserved
+	FillHistoryWithMaxPreservedFailures(history)
+	savedResults = history.List()
+	for i := uint(0); i < uint(len(savedResults)); i++ {
+		expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults)
+		if savedResults[i].debugOutput != expectedDebugOutput {
+			t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput)
+		}
+	}
+
+	// New expired failures are preserved, new success are not expired
+	FillHistoryWithMaxPreservedFailures(history)
+	FillHistoryWithMaxSuccesses(history)
+	savedResults = history.List()
+	for i := uint(0); i < uint(len(savedResults)); i++ {
+		expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults*3)
+		if savedResults[i].debugOutput != expectedDebugOutput {
+			t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput)
+		}
+	}
+}