From 03a26db636179db834f4aab9d1858b8853c59c2e Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Tue, 21 Nov 2023 10:59:20 +1030 Subject: [PATCH] x-pack/filebeat/input/cel: uppress and log max http request retry errors The retryablehttp.Client will return an error based on the ErrorHandler field when the number of requests exceeds the maximimum configuration. In the case that the field is nil, this is a non-nil error that causes issues with some APIs and does not allow the CEL code to evaluate the HTTP response status. So add a retryablehttp.ErrorHandler that will log the failure, but return the response unaltered and a nil error. --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/cel/input.go | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index f75bb1624efc..f8baf8a685b6 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -213,6 +213,7 @@ Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will d - Made Azure Blob Storage input GA and updated docs accordingly. {pull}37128[37128] - Add request trace logging to http_endpoint input. {issue}36951[36951] {pull}36957[36957] - Made GCS input GA and updated docs accordingly. {pull}37127[37127] +- Suppress and log max HTTP request retry errors in CEL input. {pull}[] *Auditbeat* diff --git a/x-pack/filebeat/input/cel/input.go b/x-pack/filebeat/input/cel/input.go index b65f0ae8d100..33a0fca4c853 100644 --- a/x-pack/filebeat/input/cel/input.go +++ b/x-pack/filebeat/input/cel/input.go @@ -730,14 +730,16 @@ func newClient(ctx context.Context, cfg config, log *logp.Logger) (*http.Client, c.CheckRedirect = checkRedirect(cfg.Resource, log) if cfg.Resource.Retry.getMaxAttempts() > 1 { + maxAttempts := cfg.Resource.Retry.getMaxAttempts() c = (&retryablehttp.Client{ HTTPClient: c, Logger: newRetryLog(log), RetryWaitMin: cfg.Resource.Retry.getWaitMin(), RetryWaitMax: cfg.Resource.Retry.getWaitMax(), - RetryMax: cfg.Resource.Retry.getMaxAttempts(), + RetryMax: maxAttempts, CheckRetry: retryablehttp.DefaultRetryPolicy, Backoff: retryablehttp.DefaultBackoff, + ErrorHandler: retryErrorHandler(maxAttempts, log), }).StandardClient() } @@ -831,6 +833,16 @@ func checkRedirect(cfg *ResourceConfig, log *logp.Logger) func(*http.Request, [] } } +// retryErrorHandler returns a retryablehttp.ErrorHandler that will log retry resignation +// but return the last retry attempt's response and a nil error so that the CEL code +// can evaluate the response status itself. +func retryErrorHandler(max int, log *logp.Logger) retryablehttp.ErrorHandler { + return func(resp *http.Response, err error, numTries int) (*http.Response, error) { + log.Warnw("giving up retries", "method", resp.Request.Method, "url", resp.Request.URL, "retries", max+1) + return resp, nil + } +} + func newRateLimiterFromConfig(cfg *ResourceConfig) *rate.Limiter { r := rate.Inf b := 1