Update queue / ES connection defaults

CHANGELOG.next.asciidoc

@@ -11,7 +11,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 *Affecting all Beats*
 - The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681]
 - Elastic-agent-autodiscover library updated to version 0.6.4, disabling metadata for deployment and cronjob. Pods that will be created from deployments or cronjobs will not have the extra metadata field for kubernetes.deployment or kubernetes.cronjob, respectively. {pull}36877[36877]
-
+- Defaults are changing for some options in the queue and Elasticsearch output, to improve typical performance based on current benchmark data. All changes can be overridden by setting them explicitly in the beat configuration. {pull}36990[36990] The changes are:
+  - `queue.mem.events` is changing from `4096` to `3200`.
+  - `queue.mem.flush.min_events` is changing from `2048` to `1600`.
+  - `queue.mem.flush.timeout` is changing from `1s` to `10s`.
+  - `output.elasticsearch.bulk_max_size` is changing from `50` to `1600`.
+  - `output.elasticsearch.idle_connection_timeout` is changing from `60s` to `3s`.
 
 
 *Auditbeat*

libbeat/cmd/instance/beat_test.go

@@ -279,7 +279,7 @@ func TestPromoteOutputQueueSettings(t *testing.T) {
 	}{
 		"blank": {
 			input:     []byte(""),
-			memEvents: 4096,
+			memEvents: 3200,
 		},
 		"defaults": {
 			input: []byte(`
@@ -289,7 +289,7 @@ output:
     hosts:
       - "localhost:9200"
 `),
-			memEvents: 4096,
+			memEvents: 3200,
 		},
 		"topLevelQueue": {
 			input: []byte(`

libbeat/docs/queueconfig.asciidoc

@@ -61,7 +61,7 @@ You can specify the following options in the `queue.mem` section of the +{beatna
 
 Number of events the queue can store. 
 
-The default value is 4096 events.
+The default value is 3200 events.
 
 [float]
 ===== `flush.min_events`
@@ -70,15 +70,15 @@ Minimum number of events required for publishing. If this value is set to 0, the
 output can start publishing events without additional waiting times. Otherwise
 the output has to wait for more events to become available.
 
-The default value is 2048.
+The default value is 1600.
 
 [float]
 ===== `flush.timeout`
 
 Maximum wait time for `flush.min_events` to be fulfilled. If set to 0s, events
 will be immediately available for consumption.
 
-The default value is 1s.
+The default value is 10s.
 
 [float]
 [[configuration-internal-queue-disk]]

libbeat/outputs/elasticsearch/config.go

@@ -54,7 +54,7 @@ type Backoff struct {
 }
 
 const (
-	defaultBulkSize = 50
+	defaultBulkSize = 1600
 )
 
 var (
@@ -74,10 +74,18 @@ var (
 			Init: 1 * time.Second,
 			Max:  60 * time.Second,
 		},
-		Transport: httpcommon.DefaultHTTPTransportSettings(),
+		Transport: esDefaultTransportSettings(),
 	}
 )
 
+func esDefaultTransportSettings() httpcommon.HTTPTransportSettings {
+	transport := httpcommon.DefaultHTTPTransportSettings()
+	// The ES output differs from the common transport settings by having
+	// a 3-second idle timeout
+	transport.IdleConnTimeout = 3 * time.Second
+	return transport
+}
+
 func (c *elasticsearchConfig) Validate() error {
 	if c.APIKey != "" && (c.Username != "" || c.Password != "") {
 		return fmt.Errorf("cannot set both api_key and username/password")

libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc

@@ -661,7 +661,7 @@ endif::[]
 
 ===== `bulk_max_size`
 
-The maximum number of events to bulk in a single Elasticsearch bulk API index request. The default is 50.
+The maximum number of events to bulk in a single Elasticsearch bulk API index request. The default is 1600.
 
 Events can be collected into batches. {beatname_uc} will split batches larger than `bulk_max_size`
 into multiple batches.
@@ -693,7 +693,7 @@ Elasticsearch after a network error. The default is `60s`.
 
 The maximum amount of time an idle connection will remain idle before closing itself.
 Zero means no limit. The format is a Go language duration (example 60s is 60 seconds).
-The default is 0.
+The default is 3s.
 
 ===== `timeout`
 

libbeat/publisher/queue/memqueue/config.go

@@ -32,9 +32,9 @@ type config struct {
 }
 
 var defaultConfig = config{
-	Events:         4 * 1024,
-	FlushMinEvents: 2 * 1024,
-	FlushTimeout:   1 * time.Second,
+	Events:         3200,
+	FlushMinEvents: 1600,
+	FlushTimeout:   10 * time.Second,
 }
 
 func (c *config) Validate() error {
@@ -53,6 +53,7 @@ func SettingsForUserConfig(cfg *c.C) (Settings, error) {
 			return Settings{}, fmt.Errorf("couldn't unpack memory queue config: %w", err)
 		}
 	}
+	//nolint:gosimple // Actually want this conversion to be explicit since the types aren't definitionally equal.
 	return Settings{
 		Events:         config.Events,
 		FlushMinEvents: config.FlushMinEvents,