Add support for HTTP header file

Resolves prometheus#980

Signed-off-by: Daniel Teunis <[email protected]>

CONFIGURATION.md

@@ -50,6 +50,10 @@ The other placeholders are specified separately.
   headers:
     [ <string>: <string> ... ]
 
+  # Read the HTTP headers from a file.
+  # It is mutually exclusive with `headers`.
+  [ headers_file: <filename> ]
+
   # The maximum uncompressed body length in bytes that will be processed. A value of 0 means no limit.
   #
   # If the response includes a Content-Length header, it is NOT validated against this value. This

config/config.go

@@ -14,6 +14,7 @@
 package config
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"math"
@@ -213,6 +214,7 @@ type HTTPProbe struct {
 	FailIfNotSSL                 bool                    `yaml:"fail_if_not_ssl,omitempty"`
 	Method                       string                  `yaml:"method,omitempty"`
 	Headers                      map[string]string       `yaml:"headers,omitempty"`
+	HeadersFile                  string                  `yaml:"headers_file,omitempty"`
 	FailIfBodyMatchesRegexp      []Regexp                `yaml:"fail_if_body_matches_regexp,omitempty"`
 	FailIfBodyNotMatchesRegexp   []Regexp                `yaml:"fail_if_body_not_matches_regexp,omitempty"`
 	FailIfHeaderMatchesRegexp    []HeaderMatch           `yaml:"fail_if_header_matches,omitempty"`
@@ -330,6 +332,18 @@ func (s *HTTPProbe) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		s.HTTPClientConfig.FollowRedirects = !*s.NoFollowRedirects
 	}
 
+	if s.HeadersFile != "" {
+		if len(s.Headers) > 0 {
+			return fmt.Errorf("setting both headers and headers_file is not allowed")
+		}
+
+		headers, err := parseKeyValueFile(s.HeadersFile)
+		if err != nil {
+			return fmt.Errorf("could not read headers file: %s", err)
+		}
+		s.Headers = headers
+	}
+
 	for key, value := range s.Headers {
 		switch textproto.CanonicalMIMEHeaderKey(key) {
 		case "Accept-Encoding":
@@ -510,3 +524,43 @@ func isCompressionAcceptEncodingValid(encoding, acceptEncoding string) bool {
 
 	return false
 }
+
+// parseKeyValueFile parses the content of a key-value file and stores it
+// in a map. The key-value file consists of lines with the following
+// structure:
+//
+// key: value
+//
+// Key and value may contain any character. Leading and trailing tabs and
+// whitespaces are stripped.
+func parseKeyValueFile(filename string) (map[string]string, error) {
+	re, err := NewRegexp(`^(?P<key>.+):(?P<value>.+)$`)
+	if err != nil {
+		return nil, fmt.Errorf("failed to compile regular expression: %s", err)
+	}
+
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open header file: %s", err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	headers := make(map[string]string)
+	for scanner.Scan() {
+		line := scanner.Text()
+		m := re.FindStringSubmatch(line)
+		if len(m) < 1 {
+			return nil, fmt.Errorf("header line could not be parsed: `%s`", line)
+		}
+		key := strings.TrimSpace(m[1])
+		value := strings.TrimSpace(m[2])
+		headers[key] = value
+	}
+
+	if err := scanner.Err(); err != nil {
+		return nil, fmt.Errorf("error during header file parsing: %s", err)
+	}
+
+	return headers, nil
+}

config/config_test.go

@@ -14,6 +14,7 @@
 package config
 
 import (
+	"reflect"
 	"strings"
 	"testing"
 
@@ -99,6 +100,18 @@ func TestLoadBadConfigs(t *testing.T) {
 			input: "testdata/invalid-tcp-query-response-regexp.yml",
 			want:  `error parsing config file: "Could not compile regular expression" regexp=":["`,
 		},
+		{
+			input: "testdata/http-headers/invalid-http-headers-config.yml",
+			want:  `error parsing config file: setting both headers and headers_file is not allowed`,
+		},
+		{
+			input: "testdata/http-headers/http-headers-file-missing.yml",
+			want:  `error parsing config file: could not read headers file: failed to open header file: open http-headers-non-existing-file.yml: no such file or directory`,
+		},
+		{
+			input: "testdata/http-headers/invalid-http-headers-file-config.yml",
+			want:  "error parsing config file: could not read headers file: header line could not be parsed: `invalid-header`",
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.input, func(t *testing.T) {
@@ -214,3 +227,41 @@ func TestIsEncodingAcceptable(t *testing.T) {
 		})
 	}
 }
+
+func TestHTTPHeaderFile(t *testing.T) {
+	sc := &SafeConfig{
+		C: &Config{},
+	}
+
+	testcases := map[string]struct {
+		input string
+		want  map[string]string
+	}{
+		"empty body": {
+			input: "testdata/http-headers/valid-empty-body-config.yml",
+			want:  map[string]string{},
+		},
+		"valid body": {
+			input: "testdata/http-headers/valid-body-config.yml",
+			want: map[string]string{
+				"Key-1": "Value 1",
+				"Key-2": "Value 2",
+				"Key-3": "Value 3",
+				"Key-4": "Value 4",
+				"Key-5": "Value 5 with trailing spaces",
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.input, func(t *testing.T) {
+			if err := sc.ReloadConfig(tc.input, nil); err != nil {
+				t.Errorf("Unexpected error: %s", err)
+			}
+			got := sc.C.Modules["http-test"].HTTP.Headers
+			if !reflect.DeepEqual(tc.want, got) {
+				t.Errorf("Unexpected HTTP Headers: want=%s got=%s", tc.want, got)
+			}
+		})
+	}
+}

config/testdata/http-headers/http-headers-file-missing.yml

@@ -0,0 +1,6 @@
+modules:
+  http_test:
+    prober: http
+    timeout: 5s
+    http:
+      headers_file: "http-headers-non-existing-file.yml"

config/testdata/http-headers/invalid-http-headers-config.yml

@@ -0,0 +1,8 @@
+modules:
+  http_test:
+    prober: http
+    timeout: 5s
+    http:
+      headers:
+        Host: host.example.com
+      headers_file: "test_headers.yml"

config/testdata/http-headers/invalid-http-headers-file-config.yml

@@ -0,0 +1,6 @@
+modules:
+  http_test:
+    prober: http
+    timeout: 5s
+    http:
+      headers_file: "testdata/http-headers/invalid-http-headers-file.yml"

config/testdata/http-headers/invalid-http-headers-file.yml

@@ -0,0 +1 @@
+invalid-header

config/testdata/http-headers/valid-body-config.yml

@@ -0,0 +1,6 @@
+modules:
+  http-test:
+    prober: http
+    timeout: 5s
+    http:
+      headers_file: "testdata/http-headers/valid-body.txt"

config/testdata/http-headers/valid-body.txt

@@ -0,0 +1,5 @@
+Key-1: Value 1
+  Key-2: Value 2
+Key-3  :Value 3
+Key-4:Value 4
+Key-5: Value 5 with trailing spaces  

config/testdata/http-headers/valid-empty-body-config.yml

@@ -0,0 +1,6 @@
+modules:
+  http-test:
+    prober: http
+    timeout: 5s
+    http:
+      headers_file: "testdata/http-headers/valid-empty-body.txt"

example.yml

@@ -33,6 +33,10 @@ modules:
     http:
       proxy_url: "http://127.0.0.1:3128"
       skip_resolve_phase_with_proxy: true
+  http_with_headers_file:
+    prober: http
+    http:
+      headers_file: "/files/http_headers.txt"
   http_post_2xx:
     prober: http
     timeout: 5s