Merge branch 'main' into remove-docker-verify-link

elastic · Sep 13, 2023 · 950ce7b · 950ce7b
2 parents 15aff09 + ad2da2b
commit 950ce7b
Show file tree

Hide file tree

Showing 16 changed files with 367 additions and 84 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,96 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.10.0]]
+=== Beats version 8.10.0
+https://github.com/elastic/beats/compare/v8.9.2\...v8.10.0[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+- Improve StreamBuf append to improve performance when reading long lines from files. {pull}35928[35928]
+- Eliminate cloning of event in deepUpdate {pull}35945[35945]
+- Fix ndjson parser to store JSON fields correctly under `target` {issue}29395[29395]
+- Add default cgroup regex for `add_process_metadata` processor {pull}36484[36484] {issue}32961[32961]
+- Fix environment capture by `add_process_metadata` processor. {issue}36469[36469] {pull}36471[36471]
+- Fix status reporting to {agent} when output configuration is invalid running under Elastic-Agent {pull}35719[35719]
+
+*Filebeat*
+
+- Fix error message formatting from filestream input. {pull}35658[35658]
+- Fixed concurrency and flaky tests issue in Azure Blob Storage input. {issue}35983[35983] {pull}36124[36124]
+- Filter out duplicate paths resolved from matching globs. {issue}36253[36253] {pull}36256[36256]
+- Remove 'onFilteredOut' and 'onDroppedOnPublish' callback logs {issue}36299[36299] {pull}36399[36399]
+- Ensure winlog input retains metric collection when handling recoverable errors. {issue}36479[36479] {pull}36483[36483]
+
+*Metricbeat*
+
+- Fix the gap in fetching forecast API metrics at the end of each month for Azure billing module  {pull}36142[36142]
+- Add option in SQL module to execute queries for all databases. {pull}35688[35688]
+- Add support for api_key authentication in elasticsearch module  {pull}36274[36274]
+- Add remaining dimensions for Azure storage account to make them available for TSDB enablement. {pull}36331[36331]
+
+*Packetbeat*
+
+- Fix panic in HTTP protocol parsing when host header has empty host part. {issue}36497[36497] {issue}36518[36518]
+
+*Winlogbeat*
+
+- Ensure event loggers retains metric collection when handling recoverable errors. {issue}36479[36479] {pull}36483[36483]
+- Fix the ability to use filtering features (e.g. `ignore_older`, `event_id`, `provider`, `level`) while reading `.evtx` files. {issue}16826[16826] {pull}36173[36173]
+
+==== Added
+
+*Affecting all Beats*
+
+- When running under Elastic-Agent the status is now reported per Unit instead of the whole Beat {issue}35874[35874] {pull}36183[36183]
+- Mark `translate_sid` processor is GA. {issue}36279[36279] {pull}36280[36280]
+- Upgrade Go to 1.20.7 {pull}36241[36241]
+
+*Auditbeat*
+
+- Add support for `security.selinux` and `system.posix_acl_access` extended attributes to FIM. {issue}36265[36265] {pull}36310[36310]
+
+*Filebeat*
+
+- Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
+- Allow specifying since when to read journald entries. {pull}35408[35408]
+- Under elastic-agent the input metrics will now be included in agent diagnostics dumps. {pull}35798[35798]
+- Improve CEL input performance. {pull}35915[35915]
+- Added support for min/max template functions in httpjson input. {issue}36094[36094] {pull}36036[36036]
+- Add `clean_session` configuration setting for MQTT input.  {pull}35806[35806]
+- Add fingerprint mode for the filestream scanner and new file identity based on it {issue}34419[34419] {pull}35734[35734]
+- Add file system metadata to events ingested via filestream {issue}35801[35801] {pull}36065[36065]
+- Add support for localstack based input integration testing {pull}35727[35727]
+- Allow parsing bytes in and bytes out as long integer in CEF processor. {issue}36100[36100] {pull}36108[36108]
+- Add support for registered owners and users to AzureAD entity analytics provider. {pull}36092[36092]
+- Added support for Okta OAuth2 provider in the httpjson input. {pull}36273[36273]
+- Add support of the interval parameter in Salesforce setupaudittrail-rest fileset. {issue}35917[35917] {pull}35938[35938]
+- Add device handling to Okta input package for entity analytics. {pull}36049[36049]
+- Add setup option `--force-enable-module-filesets`, that will act as if all filesets have been enabled in a module during setup. {issue}30915[30915] {pull}36286[36286]
+- [Azure] Add input metrics to the azure-eventhub input. {pull}35739[35739]
+
+*Metricbeat*
+
+- Add support for multiple regions in GCP {pull}32964[32964]
+- Add kubernetes.deployment.status.* fields for Kubernetes module {pull}35999[35999]
+
+*Packetbeat*
+
+- Under elastic-agent the input metrics will now be included in agent diagnostics dumps. {pull}35798[35798]
+- Add support for multiple regions in GCP {pull}32964[32964]
+
+*Winlogbeat*
+
+- Under elastic-agent the input metrics will now be included in agent diagnostics dumps. {pull}35798[35798]
+
+==== Deprecated
+
+*Heartbeat*
+
+- Deprecate aws_elb autodiscover provider. {pull}36191[36191]
+
+
 [[release-notes-8.9.2]]
 === Beats version 8.9.2
 https://github.com/elastic/beats/compare/v8.9.1\...v8.9.2[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -35,7 +35,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 *Winlogbeat*
 
 - Add "event.category" and "event.type" to Sysmon module for EventIDs 8, 9, 19, 20, 27, 28, 255 {pull}35193[35193]
-- Fix the ability to use filtering features (e.g. `ignore_older`, `event_id`, `provider`, `level`) while reading `.evtx` files. {issue}16826[16826] {pull}36173[36173]
 
 *Functionbeat*
 
@@ -52,10 +51,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix namespacing on self-monitoring {pull}32336[32336]
 - Fix Beats started by agent do not respect the allow_older_versions: true configuration flag {issue}34227[34227] {pull}34964[34964]
 - Fix performance issues when we have a lot of inputs starting and stopping by allowing to disable global processors under fleet. {issue}35000[35000] {pull}35031[35031]
-- In cases where the matcher detects a non-string type in a match statement, report the error as a debug statement, and not a warning statement. {pull}35119[35119]
 - 'add_cloud_metadata' processor - add cloud.region field for GCE cloud provider
 - 'add_cloud_metadata' processor - update azure metadata api version to get missing `cloud.account.id` field
-- Make sure k8s watchers are closed when closing k8s meta processor. {pull}35630[35630]
 - Upgraded apache arrow library used in x-pack/libbeat/reader/parquet from v11 to v12.0.1 in order to fix cross-compilation issues {pull}35640[35640]
 - Fix panic when MaxRetryInterval is specified, but RetryInterval is not {pull}35820[35820]
 - Do not print context cancelled error message when running under agent {pull}36006[36006]
@@ -124,9 +121,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Make generic SQL GA {pull}34637[34637]
 - Collect missing remote_cluster in elasticsearch ccr metricset {pull}34957[34957]
 - Add context with timeout in AWS API calls {pull}35425[35425]
-- Fix no error logs displayed in CloudWatch EC2, RDS and SQS metadata {issue}34985[34985] {pull}35035[35035]
-- Remove Beta warning from IIS application_pool metricset {pull}35480[35480]
-- Improve documentation for ActiveMQ module {issue}35113[35113] {pull}35558[35558]
 - Fix EC2 host.cpu.usage {pull}35717[35717]
 - Resolve statsd module's prematurely halting of metrics parsing upon encountering an invalid packet. {pull}35075[35075]
 - Fix the gap in fetching forecast API metrics at the end of each month for Azure billing module  {pull}36142[36142]
@@ -142,11 +136,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Packetbeat*
 
-- Fix panic in HTTP protocol parsing when host header has empty host part. {issue}36497[36497] {issue}36518[36518]
 
 *Winlogbeat*
 
-- Ensure event loggers retains metric collection when handling recoverable errors. {issue}36479[36479] {pull}36483[36483]
 
 *Elastic Logging Plugin*
 
@@ -165,7 +157,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Auditbeat*
 
-- Add support for `security.selinux` and `system.posix_acl_access` extended attributes to FIM. {issue}36265[36265] {pull}36310[36310]
 
 *Filebeat*
 
@@ -174,7 +165,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add cloudflare R2 to provider list in AWS S3 input. {pull}32620[32620]
 - Add support for single string containing multiple relation-types in getRFC5988Link. {pull}32811[32811]
 - Added separation of transform context object inside httpjson. Introduced new clause `.parent_last_response.*` {pull}33499[33499]
-- Adding filename details from zip to response for httpjson {issue}33952[33952] {pull}34044[34044]
 - Added metric `sqs_messages_waiting_gauge` for aws-s3 input. {pull}34488[34488]
 - Add nginx.ingress_controller.upstream.ip to related.ip {issue}34645[34645] {pull}34672[34672]
 - Add unix socket log parsing for nginx ingress_controller {pull}34732[34732]
@@ -225,6 +215,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - For request tracer logging in CEL and httpjson the request and response body are no longer included in `event.original`. The body is still present in `http.{request,response}.body.content`. {pull}36531[36531]
 - Added support for Okta OAuth2 provider in the CEL input. {issue}36336[36336] {pull}36521[36521]
 - Improve error logging in HTTPJSON input. {pull}36529[36529]
+- Add input metrics to http_endpoint input. {issue}36402[36402] {pull}36427[36427]
 
 *Auditbeat*
 
@@ -238,26 +229,15 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 - Add per-thread metrics to system_summary {pull}33614[33614]
 - Add GCP CloudSQL metadata {pull}33066[33066]
-- Add support for multiple regions in GCP {pull}32964[32964]
 - Add GCP Carbon Footprint metricbeat data {pull}34820[34820]
 - Add event loop utilization metric to Kibana module {pull}35020[35020]
-- Support collecting metrics from both the monitoring account and linked accounts from AWS CloudWatch. {pull}35540[35540]
-- Add new parameter `include_linked_accounts` to enable/disable metrics collection from multiple linked AWS Accounts {pull}35648[35648]
-- Migrate Azure Billing, Monitor, and Storage metricsets to the newer SDK. {pull}33585[33585]
-- Add support for float64 values parsing for statsd metrics of counter type. {pull}35099[35099]
-- Add kubernetes.deployment.status.* fields for Kubernetes module {pull}35999[35999]
-- Add Azure resource tags support to Azure Billing module {pull}36428[36428]
 
 
 *Osquerybeat*
 
 
 *Packetbeat*
 
-- Added `packetbeat.interfaces.fanout_group` to allow a Packetbeat sniffer to join an AF_PACKET fanout group. {issue}35451[35451] {pull}35453[35453]
-- Add AF_PACKET metrics. {issue}35428[35428] {pull}35489[35489]
-- Under elastic-agent the input metrics will now be included in agent diagnostics dumps. {pull}35798[35798]
-- Add support for multiple regions in GCP {pull}32964[32964]
 
 *Packetbeat*
 
@@ -270,9 +250,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Winlogbeat*
 
-- Set `host.os.type` and `host.os.family` to "windows" if not already set. {pull}35435[35435]
-- Handle empty DNS answer data in QueryResults for the Sysmon Pipeline {pull}35207[35207]
-- Under elastic-agent the input metrics will now be included in agent diagnostics dumps. {pull}35798[35798]
 
 
 *Elastic Log Driver*
@@ -289,7 +266,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Heartbeat*
 
-- Deprecate aws_elb autodiscover provider. {pull}36191[36191]
 
 
 *Metricbeat*
@@ -320,3 +296,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 
 
+
+
+
diff --git a/NOTICE.txt b/NOTICE.txt
@@ -22291,6 +22291,36 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/sergi/go-diff
+Version: v1.3.1
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/sergi/[email protected]/LICENSE:
+
+Copyright (c) 2012-2016 The go-diff Authors. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included
+in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/shirou/gopsutil/v3
 Version: v3.22.10
@@ -47464,36 +47494,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/sergi/go-diff
-Version: v1.3.1
-Licence type (autodetected): MIT
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/sergi/[email protected]/LICENSE:
-
-Copyright (c) 2012-2016 The go-diff Authors. All rights reserved.
-
-Permission is hereby granted, free of charge, to any person obtaining a
-copy of this software and associated documentation files (the "Software"),
-to deal in the Software without restriction, including without limitation
-the rights to use, copy, modify, merge, publish, distribute, sublicense,
-and/or sell copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included
-in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/shirou/gopsutil
 Version: v3.21.11+incompatible

diff --git a/filebeat/channel/runner.go b/filebeat/channel/runner.go
@@ -155,7 +155,9 @@ func newCommonConfigEditor(
 		setOptional(meta, "pipeline", config.Pipeline)
 		setOptional(fields, "fileset.name", config.Fileset)
 		setOptional(fields, "service.type", serviceType)
-		setOptional(fields, "input.type", config.Type)
+		if !clientCfg.Processing.DisableType {
+			setOptional(fields, "input.type", config.Type)
+		}
 		if config.Module != "" {
 			event := mapstr.M{"module": config.Module}
 			if config.Fileset != "" {

diff --git a/go.mod b/go.mod
@@ -217,6 +217,7 @@ require (
 	github.com/otiai10/copy v1.12.0
 	github.com/pierrec/lz4/v4 v4.1.16
 	github.com/pkg/xattr v0.4.9
+	github.com/sergi/go-diff v1.3.1
 	github.com/shirou/gopsutil/v3 v3.22.10
 	go.elastic.co/apm/module/apmelasticsearch/v2 v2.4.4
 	go.elastic.co/apm/module/apmhttp/v2 v2.4.4
@@ -340,7 +341,6 @@ require (
 	github.com/rootless-containers/rootlesskit v1.1.0 // indirect
 	github.com/sanathkr/go-yaml v0.0.0-20170819195128-ed9d249f429b // indirect
 	github.com/segmentio/asm v1.2.0 // indirect
-	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect

diff --git a/libbeat/beat/pipeline.go b/libbeat/beat/pipeline.go
@@ -128,6 +128,9 @@ type ProcessingConfig struct {
 	// is applied to events. If nil the Beat's default behavior prevails.
 	EventNormalization *bool
 
+	// Disables the addition of input.type
+	DisableType bool
+
 	// Private contains additional information to be passed to the processing
 	// pipeline builder.
 	Private interface{}

diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-8.10.0>>
 * <<release-notes-8.9.2>>
 * <<release-notes-8.9.1>>
 * <<release-notes-8.9.0>>

diff --git a/libbeat/docs/tab-widgets/install.asciidoc b/libbeat/docs/tab-widgets/install.asciidoc
@@ -79,12 +79,11 @@ endif::[]
 
 ifeval::["{release-state}"!="unreleased"]
 
-. Download the {beatname_uc} Windows zip file from the
-https://www.elastic.co/downloads/beats/{beatname_lc}[downloads page].
+. Download the {beatname_uc} Windows zip file: https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-windows-x86_64.zip
 
 . Extract the contents of the zip file into `C:\Program Files`.
 
-. Rename the +{beatname_lc}-<version>-windows+ directory to +{beatname_uc}+.
+. Rename the +{beatname_lc}-{version}-windows-x86_64+ directory to +{beatname_uc}+.
 
 . Open a PowerShell prompt as an Administrator (right-click the PowerShell icon
 and select *Run As Administrator*).

diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml
@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0-f8bc4ebc-SNAPSHOT
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0-7dab12da-SNAPSHOT
     # When extend is used it merges healthcheck.tests, see:
     # https://github.com/docker/compose/issues/8962
     # healthcheck:
@@ -31,7 +31,7 @@ services:
     - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:8.11.0-f8bc4ebc-SNAPSHOT
+    image: docker.elastic.co/logstash/logstash:8.11.0-7dab12da-SNAPSHOT
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 600
@@ -44,7 +44,7 @@ services:
       - 5055:5055
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:8.11.0-f8bc4ebc-SNAPSHOT
+    image: docker.elastic.co/kibana/kibana:8.11.0-7dab12da-SNAPSHOT
     environment:
     - "ELASTICSEARCH_USERNAME=kibana_system_user"
     - "ELASTICSEARCH_PASSWORD=testing"