Log errors from probe

main.go

@@ -47,13 +47,14 @@ import (
 var (
 	sc = config.NewSafeConfig(prometheus.DefaultRegisterer)
 
-	configFile    = kingpin.Flag("config.file", "Blackbox exporter configuration file.").Default("blackbox.yml").String()
-	timeoutOffset = kingpin.Flag("timeout-offset", "Offset to subtract from timeout in seconds.").Default("0.5").Float64()
-	configCheck   = kingpin.Flag("config.check", "If true validate the config file and then exit.").Default().Bool()
-	historyLimit  = kingpin.Flag("history.limit", "The maximum amount of items to keep in the history.").Default("100").Uint()
-	externalURL   = kingpin.Flag("web.external-url", "The URL under which Blackbox exporter is externally reachable (for example, if Blackbox exporter is served via a reverse proxy). Used for generating relative and absolute links back to Blackbox exporter itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Blackbox exporter. If omitted, relevant URL components will be derived automatically.").PlaceHolder("<url>").String()
-	routePrefix   = kingpin.Flag("web.route-prefix", "Prefix for the internal routes of web endpoints. Defaults to path of --web.external-url.").PlaceHolder("<path>").String()
-	toolkitFlags  = webflag.AddFlags(kingpin.CommandLine, ":9115")
+	configFile     = kingpin.Flag("config.file", "Blackbox exporter configuration file.").Default("blackbox.yml").String()
+	timeoutOffset  = kingpin.Flag("timeout-offset", "Offset to subtract from timeout in seconds.").Default("0.5").Float64()
+	configCheck    = kingpin.Flag("config.check", "If true validate the config file and then exit.").Default().Bool()
+	logProbeErrors = kingpin.Flag("log.probes-errors", "Log error from probe requests").Default().Bool()
+	historyLimit   = kingpin.Flag("history.limit", "The maximum amount of items to keep in the history.").Default("100").Uint()
+	externalURL    = kingpin.Flag("web.external-url", "The URL under which Blackbox exporter is externally reachable (for example, if Blackbox exporter is served via a reverse proxy). Used for generating relative and absolute links back to Blackbox exporter itself. If the URL has a path portion, it will be used to prefix all HTTP endpoints served by Blackbox exporter. If omitted, relevant URL components will be derived automatically.").PlaceHolder("<url>").String()
+	routePrefix    = kingpin.Flag("web.route-prefix", "Prefix for the internal routes of web endpoints. Defaults to path of --web.external-url.").PlaceHolder("<path>").String()
+	toolkitFlags   = webflag.AddFlags(kingpin.CommandLine, ":9115")
 
 	moduleUnknownCounter = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "blackbox_module_unknown_total",
@@ -182,7 +183,7 @@ func run() int {
 		sc.Lock()
 		conf := sc.C
 		sc.Unlock()
-		prober.Handler(w, r, conf, logger, rh, *timeoutOffset, nil, moduleUnknownCounter)
+		prober.Handler(w, r, conf, logger, rh, *timeoutOffset, nil, moduleUnknownCounter, *logProbeErrors)
 	})
 	http.HandleFunc(*routePrefix, func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/html")

prober/handler.go

@@ -43,7 +43,8 @@ var (
 )
 
 func Handler(w http.ResponseWriter, r *http.Request, c *config.Config, logger log.Logger,
-	rh *ResultHistory, timeoutOffset float64, params url.Values, moduleUnknownCounter prometheus.Counter) {
+	rh *ResultHistory, timeoutOffset float64, params url.Values, moduleUnknownCounter prometheus.Counter,
+	logProbeErrors bool) {
 
 	if params == nil {
 		params = r.URL.Query()
@@ -108,7 +109,7 @@ func Handler(w http.ResponseWriter, r *http.Request, c *config.Config, logger lo
 		}
 	}
 
-	sl := newScrapeLogger(logger, moduleName, target)
+	sl := newScrapeLogger(logger, moduleName, target, logProbeErrors)
 	level.Info(sl).Log("msg", "Beginning probe", "probe", module.Prober, "timeout_seconds", timeoutSeconds)
 
 	start := time.Now()
@@ -159,13 +160,15 @@ type scrapeLogger struct {
 	next         log.Logger
 	buffer       bytes.Buffer
 	bufferLogger log.Logger
+	logErrors    bool
 }
 
-func newScrapeLogger(logger log.Logger, module string, target string) *scrapeLogger {
+func newScrapeLogger(logger log.Logger, module string, target string, logProbeErrors bool) *scrapeLogger {
 	logger = log.With(logger, "module", module, "target", target)
 	sl := &scrapeLogger{
-		next:   logger,
-		buffer: bytes.Buffer{},
+		next:      logger,
+		buffer:    bytes.Buffer{},
+		logErrors: logProbeErrors,
 	}
 	bl := log.NewLogfmtLogger(&sl.buffer)
 	sl.bufferLogger = log.With(bl, "ts", log.DefaultTimestampUTC, "caller", log.Caller(6), "module", module, "target", target)
@@ -176,6 +179,15 @@ func (sl scrapeLogger) Log(keyvals ...interface{}) error {
 	sl.bufferLogger.Log(keyvals...)
 	kvs := make([]interface{}, len(keyvals))
 	copy(kvs, keyvals)
+
+	if sl.logErrors {
+		for i := 0; i < len(kvs); i += 2 {
+			if kvs[i] == level.Key() && kvs[i+1] == level.ErrorValue() {
+				return sl.next.Log(kvs...)
+			}
+		}
+	}
+
 	// Switch level to debug for application output.
 	for i := 0; i < len(kvs); i += 2 {
 		if kvs[i] == level.Key() {

prober/handler_test.go

@@ -59,7 +59,7 @@ func TestPrometheusTimeoutHTTP(t *testing.T) {
 
 	rr := httptest.NewRecorder()
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil)
+		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil, false)
 	})
 
 	handler.ServeHTTP(rr, req)
@@ -81,7 +81,7 @@ func TestPrometheusConfigSecretsHidden(t *testing.T) {
 	}
 	rr := httptest.NewRecorder()
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil)
+		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil, false)
 	})
 	handler.ServeHTTP(rr, req)
 
@@ -177,7 +177,7 @@ func TestHostnameParam(t *testing.T) {
 	rr := httptest.NewRecorder()
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil)
+		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil, false)
 	})
 
 	handler.ServeHTTP(rr, req)
@@ -195,7 +195,7 @@ func TestHostnameParam(t *testing.T) {
 	c.Modules["http_2xx"].HTTP.Headers["Host"] = hostname + ".something"
 
 	handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil)
+		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil, false)
 	})
 
 	rr = httptest.NewRecorder()
@@ -242,7 +242,7 @@ func TestTCPHostnameParam(t *testing.T) {
 	rr := httptest.NewRecorder()
 
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil)
+		Handler(w, r, c, log.NewNopLogger(), &ResultHistory{}, 0.5, nil, nil, false)
 	})
 
 	handler.ServeHTTP(rr, req)

prober/icmp.go

@@ -91,7 +91,7 @@ func ProbeICMP(ctx context.Context, target string, module config.Module, registr
 	dstIPAddr, lookupTime, err := chooseProtocol(ctx, module.ICMP.IPProtocol, module.ICMP.IPProtocolFallback, target, registry, logger)
 
 	if err != nil {
-		level.Warn(logger).Log("msg", "Error resolving address", "err", err)
+		level.Error(logger).Log("msg", "Error resolving address", "err", err)
 		return false
 	}
 	durationGaugeVec.WithLabelValues("resolve").Add(lookupTime)