x-pack/filebeat/input/httpjson: redact authorization headers in loggi…

…ng (#41920) (#41950)

This requires a small duplication of the mapstr.M API to workaround an
issue in that type that can result in corruption of data.

(cherry picked from commit a641687)

Co-authored-by: Dan Kortschak <[email protected]>

CHANGELOG.next.asciidoc

@@ -189,6 +189,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Improve S3 object size metric calculation to support situations where Content-Length is not available. {pull}41755[41755]
 - Fix handling of http_endpoint request exceeding memory limits. {issue}41764[41764] {pull}41765[41765]
 - Rate limiting fixes in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41583[41583]
+- Redact authorization headers in HTTPJSON debug logs. {pull}41920[41920]
 - Further rate limiting fix in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
 - Fix streaming input handling of invalid or empty websocket messages. {pull}42036[42036]
 

x-pack/filebeat/input/httpjson/input.go

@@ -15,6 +15,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"
 
@@ -33,6 +34,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/version"
 	"github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httplog"
 	"github.com/elastic/beats/v7/x-pack/filebeat/input/internal/httpmon"
+	"github.com/elastic/beats/v7/x-pack/filebeat/input/internal/private"
 	"github.com/elastic/elastic-agent-libs/logp"
 	"github.com/elastic/elastic-agent-libs/mapstr"
 	"github.com/elastic/elastic-agent-libs/monitoring"
@@ -91,6 +93,60 @@ func Plugin(log *logp.Logger, store inputcursor.StateStore) v2.Plugin {
 	}
 }
 
+type redact struct {
+	value  mapstrM
+	fields []string
+}
+
+func (r redact) MarshalLogObject(enc zapcore.ObjectEncoder) error {
+	v, err := private.Redact(r.value, "", r.fields)
+	if err != nil {
+		return fmt.Errorf("could not redact value: %v", err)
+	}
+	return v.MarshalLogObject(enc)
+}
+
+// mapstrM is a non-mutating version of mapstr.M.
+// See https://github.com/elastic/elastic-agent-libs/issues/232.
+type mapstrM mapstr.M
+
+// MarshalLogObject implements the zapcore.ObjectMarshaler interface and allows
+// for more efficient marshaling of mapstrM in structured logging.
+func (m mapstrM) MarshalLogObject(enc zapcore.ObjectEncoder) error {
+	if len(m) == 0 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		v := m[k]
+		if inner, ok := tryToMapStr(v); ok {
+			err := enc.AddObject(k, inner)
+			if err != nil {
+				return fmt.Errorf("failed to add object: %w", err)
+			}
+			continue
+		}
+		zap.Any(k, v).AddTo(enc)
+	}
+	return nil
+}
+
+func tryToMapStr(v interface{}) (mapstrM, bool) {
+	switch m := v.(type) {
+	case mapstrM:
+		return m, true
+	case map[string]interface{}:
+		return mapstrM(m), true
+	default:
+		return nil, false
+	}
+}
+
 func test(url *url.URL) error {
 	port := func() string {
 		if url.Port() != "" {

x-pack/filebeat/input/httpjson/request.go

@@ -465,7 +465,7 @@ func (rf *requestFactory) newRequest(ctx *transformContext) (transformable, erro
 		}
 	}
 
-	rf.log.Debugf("new request: %#v", req)
+	rf.log.Debugw("new request", "req", redact{value: mapstrM(req), fields: []string{"header.Authorization"}})
 
 	return req, nil
 }