From fe52c8a0cc824e144e72b7fc4504943b1bdae8f7 Mon Sep 17 00:00:00 2001 From: Jake Utley Date: Wed, 4 Sep 2019 13:51:24 -0700 Subject: [PATCH 1/6] Fix style for make Signed-off-by: Jake Utley --- history.go | 31 ++++++++++++++-- history_test.go | 99 +++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 32 ++++++++++++---- 3 files changed, 151 insertions(+), 11 deletions(-) create mode 100644 history_test.go diff --git a/history.go b/history.go index 52e02c01..3082b3a9 100644 --- a/history.go +++ b/history.go @@ -26,10 +26,12 @@ type result struct { } type resultHistory struct { - mu sync.Mutex - nextId int64 - results []*result - maxResults uint + mu sync.Mutex + nextId int64 + results []*result + maxResults uint + preservedFailedResults []*result + maxPreservedFailedResults uint } // Add a result to the history. @@ -48,6 +50,14 @@ func (rh *resultHistory) Add(moduleName, target, debugOutput string, success boo rh.results = append(rh.results, r) if uint(len(rh.results)) > rh.maxResults { + if !rh.results[0].success { + rh.preservedFailedResults = append(rh.preservedFailedResults, rh.results[0]) + if uint(len(rh.preservedFailedResults)) > rh.maxPreservedFailedResults { + preservedFailedResults := make([]*result, len(rh.preservedFailedResults)-1) + copy(preservedFailedResults, rh.preservedFailedResults[1:]) + rh.preservedFailedResults = preservedFailedResults + } + } results := make([]*result, len(rh.results)-1) copy(results, rh.results[1:]) rh.results = results @@ -62,11 +72,24 @@ func (rh *resultHistory) List() []*result { return rh.results[:] } +// ListPreservedFailures returns a list of all preserved failed results. +func (rh *resultHistory) ListPreservedFailures() []*result { + rh.mu.Lock() + defer rh.mu.Unlock() + + return rh.preservedFailedResults[:] +} + // Get returns a given result. func (rh *resultHistory) Get(id int64) *result { rh.mu.Lock() defer rh.mu.Unlock() + for _, r := range rh.preservedFailedResults { + if r.id == id { + return r + } + } for _, r := range rh.results { if r.id == id { return r diff --git a/history_test.go b/history_test.go new file mode 100644 index 00000000..e5714c1d --- /dev/null +++ b/history_test.go @@ -0,0 +1,99 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "testing" +) + +func TestHistoryKeepsLatestResults(t *testing.T) { + history := &resultHistory{maxResults: 3, maxPreservedFailedResults: 3} + for i := 0; i < 4; i++ { + history.Add("module", "target", fmt.Sprintf("result %d", i), true) + } + + savedResults := history.List() + for i := 0; i < len(savedResults); i++ { + if savedResults[i].debugOutput != fmt.Sprintf("result %d", i+1) { + t.Errorf("History contained the wrong result at index %d", i) + } + } +} + +func FillHistoryWithMaxSuccesses(h *resultHistory) { + for i := uint(0); i < h.maxResults; i++ { + h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), true) + } +} + +func FillHistoryWithMaxPreservedFailures(h *resultHistory) { + for i := uint(0); i < h.maxPreservedFailedResults; i++ { + h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), false) + } +} + +func TestHistoryPreservesExpiredFailedResults(t *testing.T) { + history := &resultHistory{maxResults: 3, maxPreservedFailedResults: 3} + + // Success are expired, no failues are expired + FillHistoryWithMaxSuccesses(history) + FillHistoryWithMaxPreservedFailures(history) + savedResults := history.List() + savedFailedResults := history.ListPreservedFailures() + if len(savedFailedResults) > 0 { + t.Errorf("Preserved failures contains failures unnecessarily.") + } + for i := uint(0); i < uint(len(savedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults) + if savedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } + + // Failures are expired, should all be preserved + FillHistoryWithMaxPreservedFailures(history) + savedResults = history.List() + savedFailedResults = history.ListPreservedFailures() + for i := uint(0); i < uint(len(savedFailedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults) + if savedFailedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } + for i := uint(0); i < uint(len(savedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults+history.maxPreservedFailedResults) + if savedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } + + // New expired failures are preserved, new success are not expired + FillHistoryWithMaxPreservedFailures(history) + FillHistoryWithMaxSuccesses(history) + savedResults = history.List() + savedFailedResults = history.ListPreservedFailures() + for i := uint(0); i < uint(len(savedFailedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults+history.maxPreservedFailedResults*2) + if savedFailedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } + for i := uint(0); i < uint(len(savedResults)); i++ { + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults+history.maxPreservedFailedResults*3) + if savedResults[i].debugOutput != expectedDebugOutput { + t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) + } + } +} diff --git a/main.go b/main.go index e4f9b75f..4b7963bc 100644 --- a/main.go +++ b/main.go @@ -46,11 +46,12 @@ var ( C: &config.Config{}, } - configFile = kingpin.Flag("config.file", "Blackbox exporter configuration file.").Default("blackbox.yml").String() - listenAddress = kingpin.Flag("web.listen-address", "The address to listen on for HTTP requests.").Default(":9115").String() - timeoutOffset = kingpin.Flag("timeout-offset", "Offset to subtract from timeout in seconds.").Default("0.5").Float64() - configCheck = kingpin.Flag("config.check", "If true validate the config file and then exit.").Default().Bool() - historyLimit = kingpin.Flag("history.limit", "The maximum amount of items to keep in the history.").Default("100").Uint() + configFile = kingpin.Flag("config.file", "Blackbox exporter configuration file.").Default("blackbox.yml").String() + listenAddress = kingpin.Flag("web.listen-address", "The address to listen on for HTTP requests.").Default(":9115").String() + timeoutOffset = kingpin.Flag("timeout-offset", "Offset to subtract from timeout in seconds.").Default("0.5").Float64() + configCheck = kingpin.Flag("config.check", "If true validate the config file and then exit.").Default().Bool() + historyLimit = kingpin.Flag("history.limit", "The maximum amount of items to keep in the history.").Default("100").Uint() + historyPreservedFailedLimit = kingpin.Flag("history.preserved-failed-limit", "The maximum amount of failed items to preserve after expiration.").Default("5").Uint() Probers = map[string]prober.ProbeFn{ "http": prober.ProbeHTTP, @@ -200,7 +201,7 @@ func run() int { kingpin.HelpFlag.Short('h') kingpin.Parse() logger := promlog.New(promlogConfig) - rh := &resultHistory{maxResults: *historyLimit} + rh := &resultHistory{maxResults: *historyLimit, maxPreservedFailedResults: *historyPreservedFailedLimit} level.Info(logger).Log("msg", "Starting blackbox_exporter", "version", version.Info()) level.Info(logger).Log("msg", "Build context", version.BuildContext()) @@ -287,7 +288,24 @@ func run() int { html.EscapeString(r.moduleName), html.EscapeString(r.target), success, r.id) } - w.Write([]byte(` + w.Write([]byte(` +

Preserved Failed Probes

+ `)) + + preservedFailedResults := rh.ListPreservedFailures() + + for i := len(preservedFailedResults) - 1; i >= 0; i-- { + r := results[i] + success := "Success" + if !r.success { + success = "Failure" + } + fmt.Fprintf(w, "", + html.EscapeString(r.moduleName), html.EscapeString(r.target), success, r.id) + } + + w.Write([]byte(`
ModuleTargetResultDebug
%s%s%sLogs
+ `)) }) From eae7c4773694d1999d6bd6a498500d9713fb1200 Mon Sep 17 00:00:00 2001 From: Jake Utley Date: Wed, 4 Sep 2019 13:51:44 -0700 Subject: [PATCH 2/6] Fix bug Signed-off-by: Jake Utley --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 4b7963bc..79469383 100644 --- a/main.go +++ b/main.go @@ -295,7 +295,7 @@ func run() int { preservedFailedResults := rh.ListPreservedFailures() for i := len(preservedFailedResults) - 1; i >= 0; i-- { - r := results[i] + r := preservedFailedResults[i] success := "Success" if !r.success { success = "Failure" From 2dc250f99b19eaa1beff15137e9f7e6de33810b0 Mon Sep 17 00:00:00 2001 From: Jake Utley Date: Thu, 5 Sep 2019 14:37:25 -0700 Subject: [PATCH 3/6] Remove maxPreservedFailedResults, merge all results into common view in List() Signed-off-by: Jake Utley --- history.go | 23 +++++++---------------- history_test.go | 28 +++++----------------------- main.go | 29 ++++++----------------------- 3 files changed, 18 insertions(+), 62 deletions(-) diff --git a/history.go b/history.go index 3082b3a9..0cec9cdd 100644 --- a/history.go +++ b/history.go @@ -26,12 +26,11 @@ type result struct { } type resultHistory struct { - mu sync.Mutex - nextId int64 - results []*result - maxResults uint - preservedFailedResults []*result - maxPreservedFailedResults uint + mu sync.Mutex + nextId int64 + results []*result + preservedFailedResults []*result + maxResults uint } // Add a result to the history. @@ -52,7 +51,7 @@ func (rh *resultHistory) Add(moduleName, target, debugOutput string, success boo if uint(len(rh.results)) > rh.maxResults { if !rh.results[0].success { rh.preservedFailedResults = append(rh.preservedFailedResults, rh.results[0]) - if uint(len(rh.preservedFailedResults)) > rh.maxPreservedFailedResults { + if uint(len(rh.preservedFailedResults)) > rh.maxResults { preservedFailedResults := make([]*result, len(rh.preservedFailedResults)-1) copy(preservedFailedResults, rh.preservedFailedResults[1:]) rh.preservedFailedResults = preservedFailedResults @@ -69,15 +68,7 @@ func (rh *resultHistory) List() []*result { rh.mu.Lock() defer rh.mu.Unlock() - return rh.results[:] -} - -// ListPreservedFailures returns a list of all preserved failed results. -func (rh *resultHistory) ListPreservedFailures() []*result { - rh.mu.Lock() - defer rh.mu.Unlock() - - return rh.preservedFailedResults[:] + return append(rh.preservedFailedResults[:], rh.results...) } // Get returns a given result. diff --git a/history_test.go b/history_test.go index e5714c1d..3fb3c49f 100644 --- a/history_test.go +++ b/history_test.go @@ -19,7 +19,7 @@ import ( ) func TestHistoryKeepsLatestResults(t *testing.T) { - history := &resultHistory{maxResults: 3, maxPreservedFailedResults: 3} + history := &resultHistory{maxResults: 3} for i := 0; i < 4; i++ { history.Add("module", "target", fmt.Sprintf("result %d", i), true) } @@ -39,22 +39,18 @@ func FillHistoryWithMaxSuccesses(h *resultHistory) { } func FillHistoryWithMaxPreservedFailures(h *resultHistory) { - for i := uint(0); i < h.maxPreservedFailedResults; i++ { + for i := uint(0); i < h.maxResults; i++ { h.Add("module", "target", fmt.Sprintf("result %d", h.nextId), false) } } func TestHistoryPreservesExpiredFailedResults(t *testing.T) { - history := &resultHistory{maxResults: 3, maxPreservedFailedResults: 3} + history := &resultHistory{maxResults: 3} // Success are expired, no failues are expired FillHistoryWithMaxSuccesses(history) FillHistoryWithMaxPreservedFailures(history) savedResults := history.List() - savedFailedResults := history.ListPreservedFailures() - if len(savedFailedResults) > 0 { - t.Errorf("Preserved failures contains failures unnecessarily.") - } for i := uint(0); i < uint(len(savedResults)); i++ { expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults) if savedResults[i].debugOutput != expectedDebugOutput { @@ -65,15 +61,8 @@ func TestHistoryPreservesExpiredFailedResults(t *testing.T) { // Failures are expired, should all be preserved FillHistoryWithMaxPreservedFailures(history) savedResults = history.List() - savedFailedResults = history.ListPreservedFailures() - for i := uint(0); i < uint(len(savedFailedResults)); i++ { - expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults) - if savedFailedResults[i].debugOutput != expectedDebugOutput { - t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) - } - } for i := uint(0); i < uint(len(savedResults)); i++ { - expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults+history.maxPreservedFailedResults) + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults) if savedResults[i].debugOutput != expectedDebugOutput { t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) } @@ -83,15 +72,8 @@ func TestHistoryPreservesExpiredFailedResults(t *testing.T) { FillHistoryWithMaxPreservedFailures(history) FillHistoryWithMaxSuccesses(history) savedResults = history.List() - savedFailedResults = history.ListPreservedFailures() - for i := uint(0); i < uint(len(savedFailedResults)); i++ { - expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults+history.maxPreservedFailedResults*2) - if savedFailedResults[i].debugOutput != expectedDebugOutput { - t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) - } - } for i := uint(0); i < uint(len(savedResults)); i++ { - expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults+history.maxPreservedFailedResults*3) + expectedDebugOutput := fmt.Sprintf("result %d", i+history.maxResults*3) if savedResults[i].debugOutput != expectedDebugOutput { t.Errorf("History contained the wrong result at index %d. Expected: %s, Actual: %s", i, expectedDebugOutput, savedResults[i].debugOutput) } diff --git a/main.go b/main.go index 79469383..79253501 100644 --- a/main.go +++ b/main.go @@ -46,12 +46,11 @@ var ( C: &config.Config{}, } - configFile = kingpin.Flag("config.file", "Blackbox exporter configuration file.").Default("blackbox.yml").String() - listenAddress = kingpin.Flag("web.listen-address", "The address to listen on for HTTP requests.").Default(":9115").String() - timeoutOffset = kingpin.Flag("timeout-offset", "Offset to subtract from timeout in seconds.").Default("0.5").Float64() - configCheck = kingpin.Flag("config.check", "If true validate the config file and then exit.").Default().Bool() - historyLimit = kingpin.Flag("history.limit", "The maximum amount of items to keep in the history.").Default("100").Uint() - historyPreservedFailedLimit = kingpin.Flag("history.preserved-failed-limit", "The maximum amount of failed items to preserve after expiration.").Default("5").Uint() + configFile = kingpin.Flag("config.file", "Blackbox exporter configuration file.").Default("blackbox.yml").String() + listenAddress = kingpin.Flag("web.listen-address", "The address to listen on for HTTP requests.").Default(":9115").String() + timeoutOffset = kingpin.Flag("timeout-offset", "Offset to subtract from timeout in seconds.").Default("0.5").Float64() + configCheck = kingpin.Flag("config.check", "If true validate the config file and then exit.").Default().Bool() + historyLimit = kingpin.Flag("history.limit", "The maximum amount of items to keep in the history.").Default("100").Uint() Probers = map[string]prober.ProbeFn{ "http": prober.ProbeHTTP, @@ -201,7 +200,7 @@ func run() int { kingpin.HelpFlag.Short('h') kingpin.Parse() logger := promlog.New(promlogConfig) - rh := &resultHistory{maxResults: *historyLimit, maxPreservedFailedResults: *historyPreservedFailedLimit} + rh := &resultHistory{maxResults: *historyLimit} level.Info(logger).Log("msg", "Starting blackbox_exporter", "version", version.Info()) level.Info(logger).Log("msg", "Build context", version.BuildContext()) @@ -288,22 +287,6 @@ func run() int { html.EscapeString(r.moduleName), html.EscapeString(r.target), success, r.id) } - w.Write([]byte(` -

Preserved Failed Probes

- `)) - - preservedFailedResults := rh.ListPreservedFailures() - - for i := len(preservedFailedResults) - 1; i >= 0; i-- { - r := preservedFailedResults[i] - success := "Success" - if !r.success { - success = "Failure" - } - fmt.Fprintf(w, "", - html.EscapeString(r.moduleName), html.EscapeString(r.target), success, r.id) - } - w.Write([]byte(`
ModuleTargetResultDebug
%s%s%sLogs
`)) From c164d6b4a4f58df6f5f1af697abfebe402704a4a Mon Sep 17 00:00:00 2001 From: Jake Utley Date: Thu, 5 Sep 2019 14:42:33 -0700 Subject: [PATCH 4/6] Remove newline to re-match origin Signed-off-by: Jake Utley --- main.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/main.go b/main.go index 79253501..e4f9b75f 100644 --- a/main.go +++ b/main.go @@ -287,8 +287,7 @@ func run() int { html.EscapeString(r.moduleName), html.EscapeString(r.target), success, r.id) } - w.Write([]byte(` - + w.Write([]byte(` `)) }) From 6459e282ac36d93289c9ea448b609c816459e368 Mon Sep 17 00:00:00 2001 From: Jake Utley Date: Mon, 9 Sep 2019 16:31:21 -0700 Subject: [PATCH 5/6] Add some comments about failed history logic. Signed-off-by: Jake Utley --- history.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/history.go b/history.go index 0cec9cdd..c4977d0e 100644 --- a/history.go +++ b/history.go @@ -25,11 +25,14 @@ type result struct { success bool } +// resultHistory contains two history slices: `results` contains most recent `maxResults` results. +// After they expire out of `results`, failures will be saved in `preservedFailedResults`. This +// ensures that we are always able to see debug information about recent failures. type resultHistory struct { mu sync.Mutex nextId int64 results []*result - preservedFailedResults []*result + preservedFailedResults []*result maxResults uint } @@ -49,6 +52,8 @@ func (rh *resultHistory) Add(moduleName, target, debugOutput string, success boo rh.results = append(rh.results, r) if uint(len(rh.results)) > rh.maxResults { + // If we are about to remove a failure, add it to the failed result history, then + // remove the oldest failed result, if needed. if !rh.results[0].success { rh.preservedFailedResults = append(rh.preservedFailedResults, rh.results[0]) if uint(len(rh.preservedFailedResults)) > rh.maxResults { @@ -68,6 +73,7 @@ func (rh *resultHistory) List() []*result { rh.mu.Lock() defer rh.mu.Unlock() + // Results in each slice are disjoint. We can simply concatenate the results. return append(rh.preservedFailedResults[:], rh.results...) } From 3e95d12d59ba7fb7a2a61ede08f29da8feb839b8 Mon Sep 17 00:00:00 2001 From: Jake Utley Date: Mon, 9 Sep 2019 16:34:24 -0700 Subject: [PATCH 6/6] Fix whitespacing Signed-off-by: Jake Utley --- history.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/history.go b/history.go index c4977d0e..95b686de 100644 --- a/history.go +++ b/history.go @@ -32,7 +32,7 @@ type resultHistory struct { mu sync.Mutex nextId int64 results []*result - preservedFailedResults []*result + preservedFailedResults []*result maxResults uint }