[Heartbeat] Capture HTTP headers

Provides progress toward: elastic/uptime#190

Captures HTTP headers and stores them in the response in the same manner
as APM Server, using canonical header names in `http.response.headers`.
Values are not indexed, just stored, in ES.

heartbeat/monitors/active/http/_meta/fields.yml

@@ -28,6 +28,11 @@
               description: >
                 List of redirects followed to arrive at final content. Last item on the list is the URL for which
                 body content is shown.
+            - name: headers
+              type: object
+              enabled: false
+              description: >
+                The canonical headers of the monitored HTTP response.
         - name: rtt
           type: group
           description: >

heartbeat/monitors/active/http/config.go

@@ -53,6 +53,7 @@ type Config struct {
 type responseConfig struct {
 	IncludeBody         string `config:"include_body"`
 	IncludeBodyMaxBytes int    `config:"include_body_max_bytes"`
+	IncludeHeaders      bool   `config:"include_headers"`
 }
 
 type checkConfig struct {
@@ -96,6 +97,7 @@ var defaultConfig = Config{
 	Response: responseConfig{
 		IncludeBody:         "on_error",
 		IncludeBodyMaxBytes: 2048,
+		IncludeHeaders: true,
 	},
 	Mode: monitors.DefaultIPSettings,
 	Check: checkConfig{

heartbeat/monitors/active/http/http_test.go

@@ -109,8 +109,7 @@ func httpBaseChecks(urlStr string) validator.Validator {
 
 func respondingHTTPChecks(url string, statusCode int) validator.Validator {
 	return lookslike.Compose(
-		httpBaseChecks(url),
-		httpBodyChecks(),
+		minimalRespondingHTTPChecks(url, statusCode),
 		lookslike.MustCompile(map[string]interface{}{
 			"http": map[string]interface{}{
 				"response.status_code":   statusCode,
@@ -121,6 +120,7 @@ func respondingHTTPChecks(url string, statusCode int) validator.Validator {
 				"rtt.write_request.us":   isdef.IsDuration,
 			},
 		}),
+		respondingHTTPHeaderChecks(),
 	)
 }
 
@@ -151,6 +151,17 @@ func respondingHTTPBodyChecks(body string) validator.Validator {
 	})
 }
 
+func respondingHTTPHeaderChecks() validator.Validator {
+	return lookslike.MustCompile(map[string]interface{}{
+		"http.response.headers": map[string]interface{}{
+			"Date": isdef.IsString,
+			"Content-Length": isdef.Optional(isdef.IsString),
+			"Content-Type": isdef.Optional(isdef.IsString),
+			"Location": isdef.Optional(isdef.IsString),
+		},
+	})
+}
+
 var upStatuses = []int{
 	// 1xx
 	http.StatusContinue,
@@ -511,7 +522,11 @@ func TestRedirect(t *testing.T) {
 				hbtest.BaseChecks("", "up", "http"),
 				hbtest.SummaryChecks(1, 0),
 				minimalRespondingHTTPChecks(testURL, 200),
+				respondingHTTPHeaderChecks(),
 				lookslike.MustCompile(map[string]interface{}{
+					// For redirects that are followed we shouldn't record this header because there's no sensible
+					// value
+					"http.response.headers.Location": isdef.KeyMissing,
 					"http.response.redirects": []string{
 						server.URL + redirectingPaths["/redirect_one"],
 						server.URL + redirectingPaths["/redirect_two"],

heartbeat/monitors/active/http/task.go

@@ -256,6 +256,18 @@ func execPing(
 		"body":        bodyFields,
 	}
 
+	if responseConfig.IncludeHeaders {
+		headerFields := common.MapStr{}
+		for canonicalHeaderKey, vals := range resp.Header {
+			if len(vals) > 1 {
+				headerFields[canonicalHeaderKey] = vals
+			} else {
+				headerFields[canonicalHeaderKey] = vals[0]
+			}
+		}
+		responseFields["headers"] = headerFields
+	}
+
 	httpFields := common.MapStr{"response": responseFields}
 
 	eventext.MergeEventFields(event, common.MapStr{"http": httpFields})