Accept HTTP headers in request parameters

Signed-off-by: anemyte <[email protected]>

README.md

@@ -100,6 +100,56 @@ scrape_configs:
       - target_label: __address__
         replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
 ```
+Example with `dns_sd_config` (probe all IPs of a domain name):
+```yml
+scrape_configs:
+  - job_name: blackbox_all
+    metrics_path: /probe
+    params:
+      module: [ http_2xx ]  # Look for a HTTP 200 response.
+    dns_sd_configs:
+      - names:
+          - example.com
+          - prometheus.io
+        type: A
+        port: 443
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+        replacement: https://$1/  # Make probe URL be like https://1.2.3.4:443/
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9115  # The blackbox exporter's real hostname:port.
+
+      # Make domain name become 'Host' header for probe requests
+      - source_labels: [__meta_dns_name]
+        target_label: __param_http_header_Host
+      # and store it in 'vhost' label
+      - source_labels: [__meta_dns_name]
+        target_label: vhost
+```
+HTTP headers for probe requests can be configured either in module configuration or via parameters (takes precedence):
+```yml
+dns_sd_configs:
+  - names:
+      - example.com
+      - prometheus.io
+    type: A
+    port: 443
+
+params:
+  module: [ http_2xx ]
+  # HTTP header names shall be prefixed with 'http_header_'; only the first value of the array is used
+  http_header_Accept-Encoding: ["gzip, deflate"]
+
+relabel_configs:
+  # hyphens (-) are not allowed in label names, use double-underscore (__) instead
+  - source_labels: [__meta_dns_name]
+    regex: prometheus\\.io
+    target_label: __param_http_header_User__Agent
+    replacement: "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.212 Safari/537.36"
+```
 
 ## Permissions
 

main.go

@@ -120,6 +120,25 @@ func probeHandler(w http.ResponseWriter, r *http.Request, c *config.Config, logg
 		return
 	}
 
+	// Parse HTTP headers from query parameters
+	if module.Prober == "http" {
+		if module.HTTP.Headers == nil {
+			module.HTTP.Headers = make(map[string]string)
+		}
+		for name, value := range params {
+			if strings.HasPrefix(name, "http_header_") {
+				// Prometheus does not allow hyphens in label names but
+				// there are HTTP headers that contain them. To overcome
+				// this hyphens can be written as double-underscore and the
+				// line below cuts 'http_header_' prefix and replaces
+				// all __ with -
+				name = strings.ReplaceAll(strings.TrimPrefix(name, "http_header_"), "__", "-")
+				level.Debug(logger).Log("msg", fmt.Sprintf("Set HTTP header: %v=%v", name, value))
+				module.HTTP.Headers[name] = value[0]
+			}
+		}
+	}
+
 	sl := newScrapeLogger(logger, moduleName, target)
 	level.Info(sl).Log("msg", "Beginning probe", "probe", module.Prober, "timeout_seconds", timeoutSeconds)
 

main_test.go

@@ -17,6 +17,7 @@ import (
 	"bytes"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
 	"testing"
 	"time"
@@ -34,6 +35,7 @@ var c = &config.Config{
 			Prober:  "http",
 			Timeout: 10 * time.Second,
 			HTTP: config.HTTPProbe{
+				IPProtocolFallback: true,
 				HTTPClientConfig: pconfig.HTTPClientConfig{
 					BearerToken: "mysecret",
 				},
@@ -190,3 +192,54 @@ func TestComputeExternalURL(t *testing.T) {
 		}
 	}
 }
+
+func TestHTTPHeaderParams(t *testing.T) {
+	headers := map[string]string{
+		"Host":            "my-secret-vhost.com",
+		"User-Agent":      "unsuspicious user",
+		"Accept-Language": "en-US",
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for key, value := range headers {
+			if strings.Title(key) == "Host" {
+				if r.Host != value {
+					t.Errorf("Unexpected host: expected %q, got %q.", value, r.Host)
+				}
+				continue
+			}
+			if got := r.Header.Get(key); got != value {
+				t.Errorf("Unexpected value of header %q: expected %q, got %q", key, value, got)
+			}
+		}
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer ts.Close()
+
+	requrl := "?debug=true&target=" + ts.URL
+	for name, value := range headers {
+		name = strings.ReplaceAll(name, "-", "__")
+		requrl = requrl + "&" + "http_header_" + name + "=" + url.QueryEscape(value)
+	}
+
+	req, err := http.NewRequest("GET", requrl, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rr := httptest.NewRecorder()
+
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		probeHandler(w, r, c, log.NewNopLogger(), &resultHistory{})
+	})
+
+	handler.ServeHTTP(rr, req)
+
+	if status := rr.Code; status != http.StatusOK {
+		t.Errorf("probe request handler returned wrong status code: %v, want %v", status, http.StatusOK)
+	}
+
+	// ts does the header check but we have to confirm whether it got any request to perform that check
+	if !strings.Contains(rr.Body.String(), "probe_success 1") {
+		t.Errorf("probe failed, response body: %v", rr.Body.String())
+	}
+}

prober/http.go

@@ -341,9 +341,20 @@ func ProbeHTTP(ctx context.Context, target string, module config.Module, registr
 
 	httpClientConfig := module.HTTP.HTTPClientConfig
 	if len(httpClientConfig.TLSConfig.ServerName) == 0 {
-		// If there is no `server_name` in tls_config, use
-		// the hostname of the target.
-		httpClientConfig.TLSConfig.ServerName = targetHost
+		// If there is no `server_name` in tls_config it makes
+		// sense to use a host header value to avoid possible
+		// TLS handshake problems.
+		changed := false
+		for name, value := range httpConfig.Headers {
+			if strings.Title(name) == "Host" {
+				httpClientConfig.TLSConfig.ServerName = value
+				changed = true
+			}
+		}
+		if !changed {
+			// Otherwise use the hostname of the target.
+			httpClientConfig.TLSConfig.ServerName = targetHost
+		}
 	}
 	client, err := pconfig.NewClientFromConfig(httpClientConfig, "http_probe", pconfig.WithKeepAlivesDisabled())
 	if err != nil {