Cherry-pick #18327 to 7.x: [Heartbeat] Capture HTTP headers

CHANGELOG.next.asciidoc

@@ -443,6 +443,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Allow a list of status codes for HTTP checks. {pull}15587[15587]
 - Add additional ECS compatible fields for TLS information. {pull}17687[17687]
+- Record HTTP response headers. {pull}18327[18327]
 
 *Heartbeat*
 

heartbeat/docs/fields.asciidoc

@@ -7281,6 +7281,18 @@ type: keyword
 
 --
 
+*`http.response.headers.*`*::
++
+--
+The canonical headers of the monitored HTTP response.
+
+
+type: object
+
+Object is not enabled.
+
+--
+
 [float]
 === rtt
 

heartbeat/docs/monitors/monitor-http.asciidoc

@@ -82,6 +82,15 @@ Example configuration:
 
 Also see <<configuration-ssl>> for a full description of the `ssl` options.
 
+
+[float]
+[[monitor-http-headers]]
+=== `headers`
+
+Controls the indexing of the HTTP response headers `http.response.body.headers` field.
+
+On by default. Set `response.include_headers` to `false` to disable.
+
 [float]
 [[monitor-http-response]]
 === `response`

heartbeat/monitors/active/http/_meta/fields.yml

@@ -28,6 +28,11 @@
               description: >
                 List of redirects followed to arrive at final content. Last item on the list is the URL for which
                 body content is shown.
+            - name: headers.*
+              type: object
+              enabled: false
+              description: >
+                The canonical headers of the monitored HTTP response.
         - name: rtt
           type: group
           description: >

heartbeat/monitors/active/http/config.go

@@ -53,6 +53,7 @@ type Config struct {
 type responseConfig struct {
 	IncludeBody         string `config:"include_body"`
 	IncludeBodyMaxBytes int    `config:"include_body_max_bytes"`
+	IncludeHeaders      bool   `config:"include_headers"`
 }
 
 type checkConfig struct {
@@ -96,6 +97,7 @@ var defaultConfig = Config{
 	Response: responseConfig{
 		IncludeBody:         "on_error",
 		IncludeBodyMaxBytes: 2048,
+		IncludeHeaders:      true,
 	},
 	Mode: monitors.DefaultIPSettings,
 	Check: checkConfig{

heartbeat/monitors/active/http/http_test.go

@@ -100,7 +100,7 @@ func checkServer(t *testing.T, handlerFunc http.HandlerFunc, useUrls bool) (*htt
 
 // The minimum response is just the URL. Only to be used for unreachable server
 // tests.
-func httpBaseChecks(urlStr string) validator.Validator {
+func urlChecks(urlStr string) validator.Validator {
 	u, _ := url.Parse(urlStr)
 	return lookslike.MustCompile(map[string]interface{}{
 		"url": wrappers.URLFields(u),
@@ -109,24 +109,28 @@ func httpBaseChecks(urlStr string) validator.Validator {
 
 func respondingHTTPChecks(url string, statusCode int) validator.Validator {
 	return lookslike.Compose(
-		httpBaseChecks(url),
-		httpBodyChecks(),
-		lookslike.MustCompile(map[string]interface{}{
-			"http": map[string]interface{}{
-				"response.status_code":   statusCode,
-				"rtt.content.us":         isdef.IsDuration,
-				"rtt.response_header.us": isdef.IsDuration,
-				"rtt.total.us":           isdef.IsDuration,
-				"rtt.validate.us":        isdef.IsDuration,
-				"rtt.write_request.us":   isdef.IsDuration,
-			},
-		}),
+		minimalRespondingHTTPChecks(url, statusCode),
+		respondingHTTPStatusAndTimingChecks(statusCode),
+		respondingHTTPHeaderChecks(),
 	)
 }
 
+func respondingHTTPStatusAndTimingChecks(statusCode int) validator.Validator {
+	return lookslike.MustCompile(map[string]interface{}{
+		"http": map[string]interface{}{
+			"response.status_code":   statusCode,
+			"rtt.content.us":         isdef.IsDuration,
+			"rtt.response_header.us": isdef.IsDuration,
+			"rtt.total.us":           isdef.IsDuration,
+			"rtt.validate.us":        isdef.IsDuration,
+			"rtt.write_request.us":   isdef.IsDuration,
+		},
+	})
+}
+
 func minimalRespondingHTTPChecks(url string, statusCode int) validator.Validator {
 	return lookslike.Compose(
-		httpBaseChecks(url),
+		urlChecks(url),
 		httpBodyChecks(),
 		lookslike.MustCompile(map[string]interface{}{
 			"http": map[string]interface{}{
@@ -151,6 +155,17 @@ func respondingHTTPBodyChecks(body string) validator.Validator {
 	})
 }
 
+func respondingHTTPHeaderChecks() validator.Validator {
+	return lookslike.MustCompile(map[string]interface{}{
+		"http.response.headers": map[string]interface{}{
+			"Date":           isdef.IsString,
+			"Content-Length": isdef.Optional(isdef.IsString),
+			"Content-Type":   isdef.Optional(isdef.IsString),
+			"Location":       isdef.Optional(isdef.IsString),
+		},
+	})
+}
+
 var upStatuses = []int{
 	// 1xx
 	http.StatusContinue,
@@ -222,43 +237,46 @@ var downStatuses = []int{
 }
 
 func TestUpStatuses(t *testing.T) {
-	for _, status := range upStatuses {
-		status := status
-		t.Run(fmt.Sprintf("Test OK HTTP status %d", status), func(t *testing.T) {
-			server, event := checkServer(t, hbtest.HelloWorldHandler(status), false)
+	for _, useURLs := range []bool{true, false} {
+		for _, status := range upStatuses {
+			status := status
 
-			testslike.Test(
-				t,
-				lookslike.Strict(lookslike.Compose(
-					hbtest.BaseChecks("127.0.0.1", "up", "http"),
-					hbtest.RespondingTCPChecks(),
-					hbtest.SummaryChecks(1, 0),
-					respondingHTTPChecks(server.URL, status),
-				)),
-				event.Fields,
-			)
-		})
+			field := "hosts"
+			if useURLs {
+				field = "urls"
+			}
+
+			testName := fmt.Sprintf("Test OK HTTP status %d using %s config field", status, field)
+			t.Run(testName, func(t *testing.T) {
+				server, event := checkServer(t, hbtest.HelloWorldHandler(status), useURLs)
+
+				testslike.Test(
+					t,
+					lookslike.Strict(lookslike.Compose(
+						hbtest.BaseChecks("127.0.0.1", "up", "http"),
+						hbtest.RespondingTCPChecks(),
+						hbtest.SummaryChecks(1, 0),
+						respondingHTTPChecks(server.URL, status),
+					)),
+					event.Fields,
+				)
+			})
+		}
 	}
 }
 
-func TestUpStatusesWithUrlsConfig(t *testing.T) {
-	for _, status := range upStatuses {
-		status := status
-		t.Run(fmt.Sprintf("Test OK HTTP status %d", status), func(t *testing.T) {
-			server, event := checkServer(t, hbtest.HelloWorldHandler(status), true)
-
-			testslike.Test(
-				t,
-				lookslike.Strict(lookslike.Compose(
-					hbtest.BaseChecks("127.0.0.1", "up", "http"),
-					hbtest.RespondingTCPChecks(),
-					hbtest.SummaryChecks(1, 0),
-					respondingHTTPChecks(server.URL, status),
-				)),
-				event.Fields,
-			)
-		})
-	}
+func TestHeadersDisabled(t *testing.T) {
+	server, event := checkServer(t, hbtest.HelloWorldHandler(200), false)
+	testslike.Test(
+		t,
+		lookslike.Strict(lookslike.Compose(
+			hbtest.BaseChecks("127.0.0.1", "up", "http"),
+			hbtest.RespondingTCPChecks(),
+			hbtest.SummaryChecks(1, 0),
+			respondingHTTPChecks(server.URL, 200),
+		)),
+		event.Fields,
+	)
 }
 
 func TestDownStatuses(t *testing.T) {
@@ -444,7 +462,7 @@ func TestConnRefusedJob(t *testing.T) {
 			hbtest.BaseChecks(ip, "down", "http"),
 			hbtest.SummaryChecks(0, 1),
 			hbtest.ErrorChecks(url, "io"),
-			httpBaseChecks(url),
+			urlChecks(url),
 		)),
 		event.Fields,
 	)
@@ -466,7 +484,7 @@ func TestUnreachableJob(t *testing.T) {
 			hbtest.BaseChecks(ip, "down", "http"),
 			hbtest.SummaryChecks(0, 1),
 			hbtest.ErrorChecks(url, "io"),
-			httpBaseChecks(url),
+			urlChecks(url),
 		)),
 		event.Fields,
 	)
@@ -511,7 +529,11 @@ func TestRedirect(t *testing.T) {
 				hbtest.BaseChecks("", "up", "http"),
 				hbtest.SummaryChecks(1, 0),
 				minimalRespondingHTTPChecks(testURL, 200),
+				respondingHTTPHeaderChecks(),
 				lookslike.MustCompile(map[string]interface{}{
+					// For redirects that are followed we shouldn't record this header because there's no sensible
+					// value
+					"http.response.headers.Location": isdef.KeyMissing,
 					"http.response.redirects": []string{
 						server.URL + redirectingPaths["/redirect_one"],
 						server.URL + redirectingPaths["/redirect_two"],
@@ -523,6 +545,44 @@ func TestRedirect(t *testing.T) {
 	}
 }
 
+func TestNoHeaders(t *testing.T) {
+	server := httptest.NewServer(hbtest.HelloWorldHandler(200))
+	defer server.Close()
+
+	configSrc := map[string]interface{}{
+		"urls":                     server.URL,
+		"response.include_headers": false,
+	}
+
+	config, err := common.NewConfigFrom(configSrc)
+	require.NoError(t, err)
+
+	jobs, _, err := create("http", config)
+	require.NoError(t, err)
+
+	sched, _ := schedule.Parse("@every 1s")
+	job := wrappers.WrapCommon(jobs, "test", "", "http", sched, time.Duration(0))[0]
+
+	event := &beat.Event{}
+	_, err = job(event)
+	require.NoError(t, err)
+
+	testslike.Test(
+		t,
+		lookslike.Strict(lookslike.Compose(
+			hbtest.BaseChecks("127.0.0.1", "up", "http"),
+			hbtest.SummaryChecks(1, 0),
+			hbtest.RespondingTCPChecks(),
+			respondingHTTPStatusAndTimingChecks(200),
+			minimalRespondingHTTPChecks(server.URL, 200),
+			lookslike.MustCompile(map[string]interface{}{
+				"http.response.headers": isdef.KeyMissing,
+			}),
+		)),
+		event.Fields,
+	)
+}
+
 func TestNewRoundTripper(t *testing.T) {
 	configs := map[string]Config{
 		"Plain":      {Timeout: time.Second},

heartbeat/monitors/active/http/task.go

@@ -256,6 +256,18 @@ func execPing(
 		"body":        bodyFields,
 	}
 
+	if responseConfig.IncludeHeaders {
+		headerFields := common.MapStr{}
+		for canonicalHeaderKey, vals := range resp.Header {
+			if len(vals) > 1 {
+				headerFields[canonicalHeaderKey] = vals
+			} else {
+				headerFields[canonicalHeaderKey] = vals[0]
+			}
+		}
+		responseFields["headers"] = headerFields
+	}
+
 	httpFields := common.MapStr{"response": responseFields}
 
 	eventext.MergeEventFields(event, common.MapStr{"http": httpFields})