x-pack/filebeat/input: Fix truncation of bodies in request tracing (#…

…42327)

When logging request traces, truncate the request/response body to 10%
of the maximum log file size.

Previously, bodies were truncated to the maximum file size, less 10kB.
10kB is a reasonable number for the other trace details, but space is
also required for encoding the body data as a JSON string value.

One example JSON body was 15% larger after encoding, but the 10kB
margin is 1% or less of the total limit. A body approaching the size
limit would typically generate a log entry that exceeded the limit.

Truncating large log entries to fit the file size limit means there may
only be one such entry per file. By truncating body data to 10% of the
file limit, we can expect to see entries for several request/response
pairs in each file.

The default maximum file size of 1MB gives a default maximum body size
of 100kB.

The behavior of request tracing for the HTTP Endpoint input is
unchanged: it always truncates request bodies to a size of 10kiB.

CHANGELOG.next.asciidoc

@@ -210,6 +210,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix Netflow Template Sharing configuration handling. {pull}42080[42080]
 - Updated websocket retry error code list to allow more scenarios to be retried which could have been missed previously. {pull}42218[42218]
 - In the `streaming` input, prevent panics on shutdown with a null check and apply a consistent namespace to contextual data in debug logs. {pull}42315[42315]
+- Fix truncation of bodies in request tracing by limiting bodies to 10% of the maximum file size. {pull}42327[42327]
 
 *Heartbeat*
 

x-pack/filebeat/docs/inputs/input-cel.asciidoc

@@ -729,6 +729,7 @@ the filename option to be deleted.
 
 This value sets the maximum size, in megabytes, the log file will reach before it is rotated. By default
 logs are allowed to reach 1MB before rotation.
+Individual request/response bodies will be truncated to 10% of this size.
 
 [float]
 ==== `resource.tracer.maxage`

x-pack/filebeat/docs/inputs/input-entity-analytics.asciidoc

@@ -1044,6 +1044,13 @@ Enabling this option compromises security and should only be used for debugging.
 To differentiate the trace files generated from different input instances, a placeholder `*` can be added to the
 filename and will be replaced with the input instance id. For Example, `http-request-trace-*.ndjson`.
 
+[float]
+==== `tracer.maxsize`
+
+This value sets the maximum size, in megabytes, the log file will reach before it is rotated. By default
+logs are allowed to reach 1MB before rotation.
+Individual request/response bodies will be truncated to 10% of this size.
+
 [float]
 ==== Metrics
 

x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc

@@ -382,6 +382,7 @@ filename and will be replaced with the input instance id. For Example, `http-req
 
 This value sets the maximum size, in megabytes, the log file will reach before it is rotated. By default
 logs are allowed to reach 1MB before rotation.
+Individual request bodies will be truncated to a maximum size of 10kiB.
 
 [float]
 ==== `tracer.maxage`

x-pack/filebeat/docs/inputs/input-httpjson.asciidoc

@@ -659,6 +659,7 @@ filename and will be replaced with the input instance id. For Example, `http-req
 
 This value sets the maximum size, in megabytes, the log file will reach before it is rotated. By default
 logs are allowed to reach 1MB before rotation.
+Individual request/response bodies will be truncated to 10% of this size.
 
 [float]
 ==== `request.tracer.maxage`

x-pack/filebeat/input/cel/input.go

@@ -785,9 +785,8 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger, reg *monitorin
 		)
 		traceLogger := zap.New(core)
 
-		const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details.
-		maxSize := cfg.Resource.Tracer.MaxSize * 1e6
-		trace = httplog.NewLoggingRoundTripper(c.Transport, traceLogger, max(0, maxSize-margin), log)
+		maxBodyLen := cfg.Resource.Tracer.MaxSize * 1e6 / 10 // 10% of file max
+		trace = httplog.NewLoggingRoundTripper(c.Transport, traceLogger, maxBodyLen, log)
 		c.Transport = trace
 	} else if cfg.Resource.Tracer != nil {
 		// We have a trace log name, but we are not enabled,

x-pack/filebeat/input/entityanalytics/provider/azuread/fetcher/graph/graph.go

@@ -455,9 +455,8 @@ func requestTrace(ctx context.Context, cli *http.Client, cfg graphConf, log *log
 	)
 	traceLogger := zap.New(core)
 
-	const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details.
-	maxSize := max(1, cfg.Tracer.MaxSize) * 1e6
-	cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, max(0, maxSize-margin), log)
+	maxBodyLen := max(1, cfg.Tracer.MaxSize) * 1e6 / 10 // 10% of file max
+	cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, maxBodyLen, log)
 	return cli
 }
 

x-pack/filebeat/input/entityanalytics/provider/jamf/jamf.go

@@ -228,9 +228,8 @@ func requestTrace(ctx context.Context, cli *http.Client, cfg conf, log *logp.Log
 	)
 	traceLogger := zap.New(core)
 
-	const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details.
-	maxSize := cfg.Tracer.MaxSize * 1e6
-	cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, max(0, maxSize-margin), log)
+	maxBodyLen := cfg.Tracer.MaxSize * 1e6 / 10 // 10% of file max
+	cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, maxBodyLen, log)
 	return cli
 }
 

x-pack/filebeat/input/entityanalytics/provider/okta/okta.go

@@ -232,9 +232,8 @@ func requestTrace(ctx context.Context, cli *http.Client, cfg conf, log *logp.Log
 	)
 	traceLogger := zap.New(core)
 
-	const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details.
-	maxSize := cfg.Tracer.MaxSize * 1e6
-	cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, max(0, maxSize-margin), log)
+	maxBodyLen := cfg.Tracer.MaxSize * 1e6 / 10 // 10% of file max
+	cli.Transport = httplog.NewLoggingRoundTripper(cli.Transport, traceLogger, maxBodyLen, log)
 	return cli
 }
 

x-pack/filebeat/input/httpjson/input.go

@@ -323,12 +323,8 @@ func newNetHTTPClient(ctx context.Context, cfg *requestConfig, log *logp.Logger,
 		)
 		traceLogger := zap.New(core)
 
-		const margin = 10e3 // 1OkB ought to be enough room for all the remainder of the trace details.
-		maxSize := cfg.Tracer.MaxSize*1e6 - margin
-		if maxSize < 0 {
-			maxSize = 0
-		}
-		netHTTPClient.Transport = httplog.NewLoggingRoundTripper(netHTTPClient.Transport, traceLogger, maxSize, log)
+		maxBodyLen := cfg.Tracer.MaxSize * 1e6 / 10 // 10% of file max
+		netHTTPClient.Transport = httplog.NewLoggingRoundTripper(netHTTPClient.Transport, traceLogger, maxBodyLen, log)
 	} else if cfg.Tracer != nil {
 		// We have a trace log name, but we are not enabled,
 		// so remove all trace logs we own.