From f2645b66a31a813d3caea65f1bd6c7107520e4d0 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 21 Nov 2023 10:59:20 +1030 Subject: [PATCH] x-pack/filebeat/input/cel: uppress and log max http request retry errors The retryablehttp.Client will return an error based on the ErrorHandler field when the number of requests exceeds the maximum configuration. In the case that the field is nil, this is a non-nil error that causes issues with some APIs and does not allow the CEL code to evaluate the HTTP response status. So add a retryablehttp.ErrorHandler that will log the failure, but return the response unaltered and a nil error. --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/cel/input.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f75bb1624efc..db3c95c9760b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -213,6 +213,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128] - Add request trace logging to http_endpoint input. {issue}36951[36951] {pull}36957[36957] - Made GCS input GA and updated docs accordingly. {pull}37127[37127] +- Suppress and log max HTTP request retry errors in CEL input. {pull}37160[37160] *Auditbeat* diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index b65f0ae8d100..33a0fca4c853 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -730,14 +730,16 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client, c.CheckRedirect = checkRedirect(cfg.Resource, log) if cfg.Resource.Retry.getMaxAttempts() > 1 { + maxAttempts := cfg.Resource.Retry.getMaxAttempts() c = (&retryablehttp.Client{ HTTPClient: c, Logger: newRetryLog(log), RetryWaitMin: cfg.Resource.Retry.getWaitMin(), RetryWaitMax: cfg.Resource.Retry.getWaitMax(), - RetryMax: cfg.Resource.Retry.getMaxAttempts(), + RetryMax: maxAttempts, CheckRetry: retryablehttp.DefaultRetryPolicy, Backoff: retryablehttp.DefaultBackoff, + ErrorHandler: retryErrorHandler(maxAttempts, log), }).StandardClient() } @@ -831,6 +833,16 @@ func checkRedirect(cfg *ResourceConfig, log *logp.Logger) func(*http.Request, [] } } +// retryErrorHandler returns a retryablehttp.ErrorHandler that will log retry resignation +// but return the last retry attempt's response and a nil error so that the CEL code +// can evaluate the response status itself. +func retryErrorHandler(max int, log *logp.Logger) retryablehttp.ErrorHandler { + return func(resp *http.Response, err error, numTries int) (*http.Response, error) { + log.Warnw("giving up retries", "method", resp.Request.Method, "url", resp.Request.URL, "retries", max+1) + return resp, nil + } +} + func newRateLimiterFromConfig(cfg *ResourceConfig) *rate.Limiter { r := rate.Inf b := 1