Merge branch 'main' into feat/check-template-bf-apply

elastic · Sep 30, 2023 · c8ddd0a · c8ddd0a
2 parents 55c982c + 8ace69f
commit c8ddd0a
Show file tree

Hide file tree

Showing 83 changed files with 16,782 additions and 13,719 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -55,6 +55,7 @@ CHANGELOG*
 /libbeat/ @elastic/elastic-agent-data-plane
 /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs
 /libbeat/management @elastic/elastic-agent-control-plane
+/libbeat/processors/cache/ @elastic/security-external-integrations
 /libbeat/processors/community_id/ @elastic/security-external-integrations
 /libbeat/processors/decode_xml/ @elastic/security-external-integrations
 /libbeat/processors/decode_xml_wineventlog/ @elastic/security-external-integrations

diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -69,6 +69,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 
 ==== Bugfixes
 
+- Fix how Prometheus histograms are calculated when percentiles are provide.{pull}36537[36537]
 - Stop using `mage:import` in community beats. This was ignoring the vendorized beats directory for some mage targets, using the code available in GOPATH, this causes inconsistencies and compilation problems if the version of the code in the GOPATH is different to the vendored one. Use of `mage:import` will continue to be unsupported in custom beats till beats is migrated to go modules, or mage supports vendored dependencies. {issue}13998[13998] {pull}14162[14162]
 - Metricbeat module builders call host parser only once when instantiating light modules. {pull}20149[20149]
 - Fix export dashboard command when running against Elastic Cloud hosted Kibana. {pull}22746[22746]
@@ -87,6 +88,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Renamed an httpjson input metric to follow naming conventions. `httpjson_interval_pages_total` was renamed to `httpjson_interval_pages` because the `_total` suffix is reserved for counters. {issue}35933[35933] {pull}36169[36169]
 - Fixed some race conditions in tests {pull}36185[36185]
 - Re-enable HTTPJSON fixed flakey test. {issue}34929[34929] {pull}36525[36525]
+- Make winlogbeat/sys/wineventlog follow the unsafe.Pointer rules. {pull}36650[36650]
 
 ==== Added
 
@@ -170,6 +172,8 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Skip dependabot updates for github.com/elastic/mito. {pull}36158[36158]
 - Add device handling to Okta API package for entity analytics. {pull}35980[35980]
 - Make Filebeat HTTPJSON input process responses sequentially. {pull}36493[36493]
+- Add initial infrastructure for a caching enrichment processor. {pull}36619[36619]
+- Add file-backed cache for cache enrichment processor. {pull}36686[36686] {pull}36696[36696]
 
 ==== Deprecated
 

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,23 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.10.2]]
+=== Beats version 8.10.2
+https://github.com/elastic/beats/compare/v8.10.1\...v8.10.2[View commits]
+
+==== Bugfixes
+
+*Packetbeat*
+
+- Prevent panic when more than one interface is configured in Fleet. {issue}36574[36574] {pull}36575[36575]
+
+==== Added
+
+*Affecting all Beats*
+
+- Upgrade Go to 1.20.8 {pull}36597[36597]
+
+
 [[release-notes-8.10.1]]
 === Beats version 8.10.1
 https://github.com/elastic/beats/compare/v8.10.0\...v8.10.1[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -9,12 +9,15 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 ==== Breaking changes
 
 *Affecting all Beats*
+- The Elasticsearch output now enables compression by default. This decreases network data usage by an average of 70-80%, in exchange for 20-25% increased CPU use and ~10% increased ingestion time. The previous default can be restored by setting the flag `compression_level: 0` under `output.elasticsearch`. {pull}36681[36681]
+
 
 *Auditbeat*
 
 
 *Filebeat*
 
+- Switch types of `log.file.device`, `log.file.inode`, `log.file.idxhi`, `log.file.idxlo` and `log.file.vol` fields to strings to better align with ECS and integrations. {pull}36697[36697]
 
 *Heartbeat*
 
@@ -134,7 +137,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Packetbeat*
 
-- Prevent panic when more than one interface is configured in fleet. {issue}36574[36574] {pull}36575[36575]
 
 *Winlogbeat*
 
@@ -146,7 +148,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Affecting all Beats*
 
-- Upgrade Go to 1.20.8 {pull}36597[36597]
 - Added append Processor which will append concrete values or values from a field to target. {issue}29934[29934] {pull}33364[33364]
 - When running under Elastic-Agent the status is now reported per Unit instead of the whole Beat {issue}35874[35874] {pull}36183[36183]
 - Add warning message to SysV init scripts for RPM-based systems that lack `/etc/rc.d/init.d/functions`. {issue}35708[35708] {pull}36188[36188]
@@ -217,7 +218,11 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - For request tracer logging in CEL and httpjson the request and response body are no longer included in `event.original`. The body is still present in `http.{request,response}.body.content`. {pull}36531[36531]
 - Added support for Okta OAuth2 provider in the CEL input. {issue}36336[36336] {pull}36521[36521]
 - Improve error logging in HTTPJSON input. {pull}36529[36529]
+- Disable warning message about ingest pipeline loading when running under Elastic Agent. {pull}36659[36659]
 - Add input metrics to http_endpoint input. {issue}36402[36402] {pull}36427[36427]
+- Update mito CEL extension library to v1.6.0. {pull}36651[36651]
+- Improve template evaluation logging for HTTPJSON input. {pull}36668[36668]
+- Add CEL partial value debug function. {pull}36652[36652]
 
 *Auditbeat*
 
@@ -238,12 +243,10 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 *Osquerybeat*
 
 
-*Packetbeat*
-
-
 *Packetbeat*
 
 - Improve efficiency of sniffers by deduplicating interface configurations. {issue}36574[36574] {pull}36576[36576]
+- Bump Windows Npcap version to v1.76. {issue}36539[36539] {pull}36549[36549]
 
 *Winlogbeat*
 
@@ -305,3 +308,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 
 
+
+
+
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -13448,11 +13448,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-elasticsearch/v8
-Version: v8.9.0
+Version: v8.10.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.9.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-elasticsearch/v8@v8.10.0/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -15600,11 +15600,11 @@ limitations under the License.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/mito
-Version: v1.5.0
+Version: v1.6.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.5.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/mito@v1.6.0/LICENSE:
 
 
                                  Apache License

diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
@@ -450,8 +450,9 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Set gzip compression level.
-  #compression_level: 0
+  # Set gzip compression level. Set to 0 to disable compression.
+  # The default is 1.
+  #compression_level: 1
 
   # Configure escaping HTML symbols in strings.
   #escape_html: false

diff --git a/auditbeat/docs/getting-started.asciidoc b/auditbeat/docs/getting-started.asciidoc
@@ -35,6 +35,10 @@ system:
 
 include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 
+The commands shown are for AMD platforms, but ARM packages are also available.
+Refer to the https://www.elastic.co/downloads/beats/{beatname_lc}[download page]
+for the full list of available packages.
+
 [float]
 [[other-installation-options]]
 ==== Other installation options

diff --git a/filebeat/beater/filebeat.go b/filebeat/beater/filebeat.go
@@ -189,7 +189,7 @@ func newBeater(b *beat.Beat, plugins PluginFactory, rawConfig *conf.C) (beat.Bea
 
 // setupPipelineLoaderCallback sets the callback function for loading pipelines during setup.
 func (fb *Filebeat) setupPipelineLoaderCallback(b *beat.Beat) error {
-	if b.Config.Output.Name() != "elasticsearch" {
+	if b.Config.Output.Name() != "elasticsearch" && !b.Manager.Enabled() {
 		logp.Warn(pipelinesWarning)
 		return nil
 	}

diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc
@@ -41,6 +41,10 @@ system:
 
 include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 
+The commands shown are for AMD platforms, but ARM packages are also available.
+Refer to the https://www.elastic.co/downloads/beats/{beatname_lc}[download page]
+for the full list of available packages.
+
 [float]
 [[other-installation-options]]
 ==== Other installation options

diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -1546,8 +1546,9 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Set gzip compression level.
-  #compression_level: 0
+  # Set gzip compression level. Set to 0 to disable compression.
+  # The default is 1.
+  #compression_level: 1
 
   # Configure escaping HTML symbols in strings.
   #escape_html: false

diff --git a/go.mod b/go.mod
@@ -205,8 +205,8 @@ require (
 	github.com/elastic/elastic-agent-libs v0.3.15-0.20230913212237-dbdaf18c898b
 	github.com/elastic/elastic-agent-shipper-client v0.5.1-0.20230228231646-f04347b666f3
 	github.com/elastic/elastic-agent-system-metrics v0.6.1
-	github.com/elastic/go-elasticsearch/v8 v8.9.0
-	github.com/elastic/mito v1.5.0
+	github.com/elastic/go-elasticsearch/v8 v8.10.0
+	github.com/elastic/mito v1.6.0
 	github.com/elastic/toutoumomoma v0.0.0-20221026030040-594ef30cb640
 	github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15
 	github.com/google/cel-go v0.15.3

diff --git a/go.sum b/go.sum
@@ -667,8 +667,8 @@ github.com/elastic/fsevents v0.0.0-20181029231046-e1d381a4d270/go.mod h1:Msl1pdb
 github.com/elastic/glog v1.0.1-0.20210831205241-7d8b5c89dfc4/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/elastic/go-concert v0.2.0 h1:GAQrhRVXprnNjtvTP9pWJ1d4ToEA4cU5ci7TwTa20xg=
 github.com/elastic/go-concert v0.2.0/go.mod h1:HWjpO3IAEJUxOeaJOWXWEp7imKd27foxz9V5vegC/38=
-github.com/elastic/go-elasticsearch/v8 v8.9.0 h1:8xtmYjUkqtahl50E0Bg/wjKI7K63krJrrLipbNj/fCU=
-github.com/elastic/go-elasticsearch/v8 v8.9.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE=
+github.com/elastic/go-elasticsearch/v8 v8.10.0 h1:ALg3DMxSrx07YmeMNcfPf7cFh1Ep2+Qa19EOXTbwr2k=
+github.com/elastic/go-elasticsearch/v8 v8.10.0/go.mod h1:NGmpvohKiRHXI0Sw4fuUGn6hYOmAXlyCphKpzVBiqDE=
 github.com/elastic/go-libaudit/v2 v2.3.3 h1:PO+9/HDSn65UAyydkkoTf81QET14fWmocHFiGEX/E6M=
 github.com/elastic/go-libaudit/v2 v2.3.3/go.mod h1:+ZE0czqmbqtnRkl0fNgpI+HvVVRo/ZMJdcXv/PaKcOo=
 github.com/elastic/go-licenser v0.4.1 h1:1xDURsc8pL5zYT9R29425J3vkHdt4RT5TNEMeRN48x4=
@@ -695,8 +695,8 @@ github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a h1:8WfL/X6fK11
 github.com/elastic/gopacket v1.1.20-0.20211202005954-d412fca7f83a/go.mod h1:riddUzxTSBpJXk3qBHtYr4qOhFhT6k/1c0E3qkQjQpA=
 github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
 github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
-github.com/elastic/mito v1.5.0 h1:637UzhwJH8XfHgusGrpL9b7sTkDE+gJ4unf1tDPDtUE=
-github.com/elastic/mito v1.5.0/go.mod h1:J0LW+SbpiAoiBUBEBrbH8epwNDFhWWgEWyR/9DpY04c=
+github.com/elastic/mito v1.6.0 h1:0Gf0qovzgBCKfGoPKdqdGKBjS9L3mRREWNqb7zG2WYg=
+github.com/elastic/mito v1.6.0/go.mod h1:J0LW+SbpiAoiBUBEBrbH8epwNDFhWWgEWyR/9DpY04c=
 github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3 h1:ChPwRVv1RR4a0cxoGjKcyWjTEpxYfm5gydMIzo32cAw=
 github.com/elastic/ristretto v0.1.1-0.20220602190459-83b0895ca5b3/go.mod h1:RAy2GVV4sTWVlNMavv3xhLsk18rxhfhDnombTe6EF5c=
 github.com/elastic/sarama v1.19.1-0.20220310193331-ebc2b0d8eef3 h1:FzA0/n4iMt8ojGDGRoiFPSHFvvdVIvxOxyLtiFnrLBM=

diff --git a/heartbeat/docs/getting-started.asciidoc b/heartbeat/docs/getting-started.asciidoc
@@ -38,6 +38,10 @@ system:
 
 include::{libbeat-dir}/tab-widgets/install-widget.asciidoc[]
 
+The commands shown are for AMD platforms, but ARM packages are also available.
+Refer to the https://www.elastic.co/downloads/beats/{beatname_lc}[download page]
+for the full list of available packages.
+
 [float]
 [[other-installation-options]]
 ==== Other installation options

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -542,8 +542,9 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Set gzip compression level.
-  #compression_level: 0
+  # Set gzip compression level. Set to 0 to disable compression.
+  # The default is 1.
+  #compression_level: 1
 
   # Configure escaping HTML symbols in strings.
   #escape_html: false

diff --git a/heartbeat/monitors/wrappers/summarizer/plugstatestat.go b/heartbeat/monitors/wrappers/summarizer/plugstatestat.go
@@ -63,13 +63,15 @@ func (ssp *BrowserStateStatusPlugin) BeforeSummary(event *beat.Event) BeforeSumm
 	}
 
 	res := ssp.cssp.BeforeSummary(event)
-
+	// Browsers don't set this prior, so we set this here, as opposed to lightweight monitors
 	_, _ = event.PutValue("monitor.status", string(ssp.cssp.js.Status))
+
+	_, _ = event.PutValue("synthetics", mapstr.M{"type": "heartbeat/summary"})
 	return res
 }
 
 func (ssp *BrowserStateStatusPlugin) BeforeRetry() {
-	// noop
+	ssp.cssp.BeforeRetry()
 }
 
 // LightweightStateStatusPlugin encapsulates the writing of the primary fields used by the summary,
@@ -108,7 +110,7 @@ func (ssp *LightweightStateStatusPlugin) BeforeSummary(event *beat.Event) Before
 }
 
 func (ssp *LightweightStateStatusPlugin) BeforeRetry() {
-	// noop
+	ssp.cssp.BeforeRetry()
 }
 
 type commonSSP struct {
@@ -162,21 +164,19 @@ func (ssp *commonSSP) BeforeSummary(event *beat.Event) BeforeSummaryActions {
 		"summary": &jsCopy,
 		"state":   ms,
 	}
-	if ssp.sf.Type == "browser" {
-		fields["synthetics"] = mapstr.M{"type": "heartbeat/summary"}
-	}
-	eventext.MergeEventFields(event, fields)
 
-	if retry {
-		// mutate the js into the state for the next attempt
-		ssp.js.BumpAttempt()
-	}
+	eventext.MergeEventFields(event, fields)
 
-	logp.L().Debugf("attempt info: %v == %v && %d < %d", ssp.js.Status, lastStatus, ssp.js.Attempt, ssp.js.MaxAttempts)
+	logp.L().Debugf("attempt info: current(%v) == lastStatus(%v) && attempts(%d < %d)", ssp.js.Status, lastStatus, ssp.js.Attempt, ssp.js.MaxAttempts)
 
 	if retry {
 		return RetryBeforeSummary
 	}
 
 	return 0
 }
+
+func (ssp *commonSSP) BeforeRetry() {
+	// mutate the js into the state for the next attempt
+	ssp.js.BumpAttempt()
+}
diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl
@@ -9,8 +9,9 @@ output.elasticsearch:
   # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
   hosts: ["localhost:9200"]
 
-  # Set gzip compression level.
-  #compression_level: 0
+  # Set gzip compression level. Set to 0 to disable compression.
+  # The default is 1.
+  #compression_level: 1
 
   # Configure escaping HTML symbols in strings.
   #escape_html: false

diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-8.10.2>>
 * <<release-notes-8.10.1>>
 * <<release-notes-8.10.0>>
 * <<release-notes-8.9.2>>

diff --git a/libbeat/outputs/elasticsearch/config.go b/libbeat/outputs/elasticsearch/config.go
@@ -65,7 +65,7 @@ var (
 		Password:         "",
 		APIKey:           "",
 		MaxRetries:       3,
-		CompressionLevel: 0,
+		CompressionLevel: 1,
 		EscapeHTML:       false,
 		Kerberos:         nil,
 		LoadBalance:      true,

diff --git a/libbeat/outputs/elasticsearch/config_test.go b/libbeat/outputs/elasticsearch/config_test.go
@@ -97,6 +97,28 @@ non_indexable_policy.dead_letter_index:
 	}
 }
 
+func TestCompressionIsOnByDefault(t *testing.T) {
+	config := ""
+	c := conf.MustNewConfigFrom(config)
+	elasticsearchOutputConfig, err := readConfig(c)
+	if err != nil {
+		t.Fatalf("Can't create test configuration from valid input")
+	}
+	assert.Equal(t, 1, elasticsearchOutputConfig.CompressionLevel, "Default compression level should be 1")
+}
+
+func TestExplicitCompressionLevelOverridesDefault(t *testing.T) {
+	config := `
+compression_level: 0
+`
+	c := conf.MustNewConfigFrom(config)
+	elasticsearchOutputConfig, err := readConfig(c)
+	if err != nil {
+		t.Fatalf("Can't create test configuration from valid input")
+	}
+	assert.Equal(t, 0, elasticsearchOutputConfig.CompressionLevel, "Explicit compression level should override defaults")
+}
+
 func readConfig(cfg *conf.C) (*elasticsearchConfig, error) {
 	c := defaultConfig
 	if err := cfg.Unpack(&c); err != nil {

diff --git a/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc b/libbeat/outputs/elasticsearch/docs/elasticsearch.asciidoc
@@ -105,7 +105,7 @@ The compression level must be in the range of `1` (best speed) to `9` (best comp
 
 Increasing the compression level will reduce the network usage but will increase the cpu usage.
 
-The default value is `0`.
+The default value is `1`.
 
 ===== `escape_html`