Merge remote-tracking branch 'upstream/main' into feat/drop-go-daemon

.buildkite/packaging.pipeline.yml

@@ -111,7 +111,6 @@ steps:
           - x-pack/auditbeat
           - x-pack/dockerlogbeat
           - x-pack/filebeat
-          - x-pack/functionbeat
           - x-pack/heartbeat
           - x-pack/metricbeat
           - x-pack/osquerybeat
@@ -200,7 +199,6 @@ steps:
           - x-pack/auditbeat
           - x-pack/dockerlogbeat
           - x-pack/filebeat
-          - x-pack/functionbeat
           - x-pack/heartbeat
           - x-pack/metricbeat
           - x-pack/osquerybeat
@@ -269,7 +267,7 @@ steps:
           - packaging-snapshot
           - dashboards-snapshot
         command: |
-          buildkite-agent artifact download "build/**/*" . 
+          buildkite-agent artifact download "build/**/*" .
           .buildkite/scripts/packaging/prepare-release-manager.sh snapshot
           .buildkite/scripts/dra.sh
         agents:

.github/CODEOWNERS

@@ -26,9 +26,9 @@ CHANGELOG*
 /.github/CODEOWNERS @elastic/beats-tech-leads
 /auditbeat/ @elastic/sec-linux-platform
 /deploy/ @elastic/elastic-agent-data-plane
-/deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/obs-cloudnative-monitoring
+/deploy/kubernetes @elastic/elastic-agent-data-plane @elastic/elastic-agent-control-plane
 /dev-tools/ @elastic/elastic-agent-data-plane
-/dev-tools/kubernetes @elastic/obs-ds-hosted-services
+/dev-tools/kubernetes @elastic/elastic-agent-data-plane @elastic/elastic-agent-control-plane
 /docs/ @elastic/elastic-agent-data-plane
 /filebeat @elastic/elastic-agent-data-plane
 /filebeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
@@ -57,10 +57,11 @@ CHANGELOG*
 /heartbeat/ @elastic/obs-ds-hosted-services
 /journalbeat @elastic/elastic-agent-data-plane
 /libbeat/ @elastic/elastic-agent-data-plane
+/libbeat/autodiscover/providers/kubernetes @elastic/elastic-agent-data-plane @elastic/elastic-agent-control-plane
 /libbeat/docs/processors-list.asciidoc @elastic/ingest-docs
 /libbeat/management @elastic/elastic-agent-control-plane
-/libbeat/processors/add_cloud_metadata @elastic/obs-cloud-monitoring
-/libbeat/processors/add_kubernetes_metadata @elastic/obs-cloudnative-monitoring
+/libbeat/processors/add_cloud_metadata @elastic/obs-ds-hosted-services
+/libbeat/processors/add_kubernetes_metadata @elastic/elastic-agent-data-plane
 /libbeat/processors/cache/ @elastic/security-service-integrations
 /libbeat/processors/community_id/ @elastic/sec-deployment-and-devices
 /libbeat/processors/decode_xml/ @elastic/security-service-integrations
@@ -105,7 +106,6 @@ CHANGELOG*
 /metricbeat/module/system/ @elastic/elastic-agent-data-plane
 /metricbeat/module/vsphere @elastic/obs-infraobs-integrations
 /metricbeat/module/zookeeper @elastic/obs-infraobs-integrations
-/metricbeat/tests @elastic/ingest-eng-prod
 /packetbeat/ @elastic/sec-linux-platform
 /script/ @elastic/elastic-agent-data-plane
 /testing/ @elastic/elastic-agent-data-plane
@@ -116,10 +116,10 @@ CHANGELOG*
 /x-pack/filebeat @elastic/elastic-agent-data-plane
 /x-pack/filebeat/docs/ # Listed without an owner to avoid maintaining doc ownership for each input and module.
 /x-pack/filebeat/docs/inputs/input-salesforce.asciidoc @elastic/obs-infraobs-integrations
-/x-pack/filebeat/input/awscloudwatch/ @elastic/obs-cloud-monitoring
-/x-pack/filebeat/input/awss3/ @elastic/obs-cloud-monitoring
+/x-pack/filebeat/input/awscloudwatch/ @elastic/obs-ds-hosted-services
+/x-pack/filebeat/input/awss3/ @elastic/obs-ds-hosted-services
 /x-pack/filebeat/input/azureblobstorage/ @elastic/security-service-integrations
-/x-pack/filebeat/input/azureeventhub/ @elastic/obs-cloud-monitoring
+/x-pack/filebeat/input/azureeventhub/ @elastic/obs-ds-hosted-services
 /x-pack/filebeat/input/cel/ @elastic/security-service-integrations
 /x-pack/filebeat/input/cometd/ @elastic/obs-infraobs-integrations
 /x-pack/filebeat/input/entityanalytics/ @elastic/security-service-integrations
@@ -137,9 +137,9 @@ CHANGELOG*
 /x-pack/filebeat/input/salesforce @elastic/obs-infraobs-integrations
 /x-pack/filebeat/input/streaming/ @elastic/security-service-integrations
 /x-pack/filebeat/module/activemq @elastic/obs-infraobs-integrations
-/x-pack/filebeat/module/aws @elastic/obs-cloud-monitoring
-/x-pack/filebeat/module/awsfargate @elastic/obs-cloud-monitoring
-/x-pack/filebeat/module/azure @elastic/obs-cloud-monitoring
+/x-pack/filebeat/module/aws @elastic/obs-ds-hosted-services
+/x-pack/filebeat/module/awsfargate @elastic/obs-ds-hosted-services
+/x-pack/filebeat/module/azure @elastic/obs-ds-hosted-services
 /x-pack/filebeat/module/barracuda @elastic/security-service-integrations
 /x-pack/filebeat/module/bluecoat @elastic/sec-deployment-and-devices
 /x-pack/filebeat/module/cef @elastic/sec-deployment-and-devices
@@ -223,6 +223,7 @@ CHANGELOG*
 /x-pack/metricbeat/module/iis @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/istio/ @elastic/obs-cloudnative-monitoring
 /x-pack/metricbeat/module/mssql @elastic/obs-infraobs-integrations
+/x-pack/metricbeat/module/openai @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/oracle @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/panw @elastic/obs-infraobs-integrations
 /x-pack/metricbeat/module/prometheus/ @elastic/obs-cloudnative-monitoring

.github/workflows/updatecli.d/bump-golang-7.17.yml

@@ -157,16 +157,6 @@ targets:
         keyword: "FROM"
         matcher: "golang"
       file: ./packetbeat/Dockerfile
-  update-functionbeat-dockerfile:
-    name: "Update Functionbeat Dockerfile"
-    sourceid: latestGoVersion
-    scmid: githubConfig
-    kind: dockerfile
-    spec:
-      instruction:
-        keyword: "FROM"
-        matcher: "golang"
-      file: ./x-pack/functionbeat/Dockerfile
   update-nats-module-dockerfile:
     name: "Update NATS module Dockerfile"
     sourceid: latestGoVersion

.github/workflows/updatecli.d/bump-golang.yml

@@ -166,16 +166,6 @@ targets:
         keyword: "FROM"
         matcher: "golang"
       file: ./packetbeat/Dockerfile
-  update-functionbeat-dockerfile:
-    name: "Update Functionbeat Dockerfile"
-    sourceid: latestGoVersion
-    scmid: githubConfig
-    kind: dockerfile
-    spec:
-      instruction:
-        keyword: "FROM"
-        matcher: "golang"
-      file: ./x-pack/functionbeat/Dockerfile
   update-nats-module-dockerfile:
     name: "Update NATS module Dockerfile"
     sourceid: latestGoVersion

.gitignore

@@ -11,7 +11,6 @@
 *beat/logs
 *beat/data
 **/ironbank/build/
-x-pack/functionbeat/pkg
 
 # Files
 .DS_Store
@@ -24,8 +23,6 @@ beat.db
 *.keystore
 go_env.properties
 mage_output_file.go
-x-pack/functionbeat/*/fields.yml
-x-pack/functionbeat/provider/*/functionbeat-*
 x-pack/dockerlogbeat/temproot.tar
 
 # Editor swap files

CHANGELOG-developer.next.asciidoc

@@ -213,6 +213,7 @@ The list below covers the major changes between 7.0.0-rc2 and main only.
 - Add field redaction package. {pull}40997[40997]
 - Add support for marked redaction to x-pack/filebeat/input/internal/private {pull}41212[41212]
 - Add support for collecting Okta role and factor data for users with filebeat entityanalytics input. {pull}41044[41044]
+- Add CEL input program evaluation coverage collection support. {pull}41884[41884]
 
 ==== Deprecated
 

CHANGELOG.asciidoc

@@ -441,6 +441,11 @@ filebeat.inputs:
 === Beats version 8.14.3
 https://github.com/elastic/beats/compare/v8.14.2\...v8.14.3[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Bugfixes
 
 *Filebeat*
@@ -471,6 +476,11 @@ https://github.com/elastic/beats/compare/v8.14.2\...v8.14.3[View commits]
 === Beats version 8.14.2
 https://github.com/elastic/beats/compare/v8.14.1\...v8.14.2[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Breaking changes
 
 *Filebeat*
@@ -507,6 +517,11 @@ https://github.com/elastic/beats/compare/v8.14.1\...v8.14.2[View commits]
 === Beats version 8.14.1
 https://github.com/elastic/beats/compare/v8.14.0\...v8.14.1[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Bugfixes
 
 *Heartbeat*
@@ -518,6 +533,11 @@ https://github.com/elastic/beats/compare/v8.14.0\...v8.14.1[View commits]
 === Beats version 8.14.0
 https://github.com/elastic/beats/compare/v8.13.4\...v8.14.0[View commits]
 
+==== Known Issues
+
+*Filebeat*
+ - Filestream input will resend files that have been inactive for 30min or more. Workaround: set `clean_inactive` to a very high value, like 5 years: `clean_inactive: 43800h0m0s`. {issue}40178[40178]
+
 ==== Breaking changes
 
 *Filebeat*
@@ -2911,6 +2931,7 @@ https://github.com/elastic/beats/compare/v7.17.0\...v8.0.0[View commits]
 - Add `while_pattern` type to multiline reader. {pull}19662[19662]
 - auditd dataset: Use `process.args` to store program arguments instead of `auditd.log.aNNN` fields. {pull}29601[29601]
 - Remove deprecated old `awscloudwatch` input name. {pull}29844[29844]
+- Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817]
 
 *Metricbeat*
 

CHANGELOG.next.asciidoc

@@ -50,8 +50,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Remove deprecated awscloudwatch field from Filebeat. {pull}41089[41089]
 - The performance of ingesting SQS data with the S3 input has improved by up to 60x for queues with many small events. `max_number_of_messages` config for SQS mode is now ignored, as the new design no longer needs a manual cap on messages. Instead, use `number_of_workers` to scale ingestion rate in both S3 and SQS modes. The increased efficiency may increase network bandwidth consumption, which can be throttled by lowering `number_of_workers`. It may also increase number of events stored in memory, which can be throttled by lowering the configured size of the internal queue. {pull}40699[40699]
 - Fixes filestream logging the error "filestream input with ID 'ID' already exists, this will lead to data duplication[...]" on Kubernetes when using autodiscover. {pull}41585[41585]
-
 - Add kafka compression support for ZSTD.
+- Filebeat fails to start if there is any input with a duplicated ID. It logs the duplicated IDs and the offending inputs configurations. {pull}41731[41731]
 
 *Heartbeat*
 
@@ -115,13 +115,20 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Ensure Elasticsearch output can always recover from network errors {pull}40794[40794]
 - Add `translate_ldap_attribute` processor. {pull}41472[41472]
 - Remove unnecessary debug logs during idle connection teardown {issue}40824[40824]
+- Remove unnecessary reload for Elastic Agent managed beats when apm tracing config changes from nil to nil {pull}41794[41794]
 - Fix incorrect cloud provider identification in add_cloud_metadata processor using provider priority mechanism {pull}41636[41636]
+- Prevent panic if libbeat processors are loaded more than once. {issue}41475[41475] {pull}41857[51857]
+- Allow network condition to handle field values that are arrays of IP addresses. {pull}41918[41918]
+- Fix a bug where log files are rotated on startup when interval is configured and rotateonstartup is disabled {issue}41894[41894] {pull}41895[41895]
 
 *Auditbeat*
 
 - auditd: Request status from a separate socket to avoid data congestion {pull}41207[41207]
 - auditd: Use ECS `event.type: end` instead of `stop` for SERVICE_STOP, DAEMON_ABORT, and DAEMON_END messages. {pull}41558[41558]
 - auditd: Update syscall names for Linux 6.11. {pull}41558[41558]
+- hasher: Geneneral improvements and fixes. {pull}41863[41863]
+- hasher: Add a cached hasher for upcoming backend. {pull}41952[41952]
+- Split common tty definitions. {pull}42004[42004]
 
 *Filebeat*
 
@@ -185,6 +192,9 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix missing key in streaming input logging. {pull}41600[41600]
 - Improve S3 object size metric calculation to support situations where Content-Length is not available. {pull}41755[41755]
 - Fix handling of http_endpoint request exceeding memory limits. {issue}41764[41764] {pull}41765[41765]
+- Rate limiting fixes in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41583[41583]
+- Redact authorization headers in HTTPJSON debug logs. {pull}41920[41920]
+- Further rate limiting fix in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
 
 *Heartbeat*
 
@@ -217,6 +227,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix Kubernetes metadata sometimes not being present after startup {pull}41216[41216]
 - Do not report non-existant 0 values for RSS metrics in docker/memory {pull}41449[41449]
 - Log Cisco Meraki `getDevicePerformanceScores` errors without stopping metrics collection. {pull}41622[41622]
+- Don't skip first bucket value in GCP metrics metricset for distribution type metrics {pull}41822[41822]
 
 
 *Osquerybeat*
@@ -261,6 +272,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add linux capabilities to processes in the system/process. {pull}37453[37453]
 - Add linux capabilities to processes in the system/process. {pull}37453[37453]
 - Add process.entity_id, process.group.name and process.group.id in add_process_metadata processor. Make fim module with kprobes backend to always add an appropriately configured add_process_metadata processor to enrich file events {pull}38776[38776]
+- Split module/system/process into common and provider bits. {pull}41868[41868]
 
 *Auditbeat*
 
@@ -346,7 +358,17 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Add support for Journald in the System module. {pull}41555[41555]
 - Add ability to remove request trace logs from http_endpoint input. {pull}40005[40005]
 - Add ability to remove request trace logs from entityanalytics input. {pull}40004[40004]
+- Refactor & cleanup with updates to default values and documentation. {pull}41834[41834]
 - Update CEL mito extensions to v1.16.0. {pull}41727[41727]
+- Add `unifiedlogs` input for MacOS. {pull}41791[41791]
+- Add evaluation state dump debugging option to CEL input. {pull}41335[41335]
+- Added support for retry configuration in GCS input. {issue}11580[11580] {pull}41862[41862]
+- Improve S3 polling mode states registry when using list prefix option. {pull}41869[41869]
+- Add support for SSL and Proxy configurations for websoket type in streaming input. {pull}41934[41934]
+- AWS S3 input registry cleanup for untracked s3 objects. {pull}41694[41694]
+- The environment variable `BEATS_AZURE_EVENTHUB_INPUT_TRACING_ENABLED: true` enables internal logs tracer for the azure-eventhub input. {issue}41931[41931] {pull}41932[41932]
+- Rate limiting operability improvements in the Okta provider of the Entity Analytics input. {issue}40106[40106] {pull}41977[41977]
+- Added default values in the streaming input for websocket retries and put a cap on retry wait time to be lesser than equal to the maximum defined wait time. {pull}42012[42012]
 
 *Auditbeat*
 
@@ -397,8 +419,12 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Bump aerospike-client-go to version v7.7.1 and add support for basic auth in Aerospike module {pull}41233[41233]
 - Only watch metadata for ReplicaSets in metricbeat k8s module {pull}41289[41289]
 - Add support for region/zone for Vertex AI service in GCP module {pull}41551[41551]
+- Add support for location label as an optional configuration parameter in GCP metrics metricset. {issue}41550[41550] {pull}41626[41626]
+- Added `tier_preference`, `creation_date` and `version` fields to the `elasticsearch.index` metricset. {pull}41944[41944]
+- Add `use_performance_counters` to collect CPU metrics using performance counters on Windows for `system/cpu` and `system/core` {pull}41965[41965]
 
 *Metricbeat*
+- Add benchmark module {pull}41801[41801]
 
 
 *Osquerybeat*
@@ -417,10 +443,13 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Language setting also added to decode xml wineventlog processor {pull}41525[41525]
 - Format embedded messages in the experimental api {pull}41525[41525]
 - Implement exclusion range support for event_id. {issue}38623[38623] {pull}41639[41639]
+- Make the experimental API GA and rename it to winlogbeat-raw {issue}39580[39580] {pull}41770[41770]
 
 
 *Functionbeat*
 
+- Removal of functionbeat binaries from CI pipelines {issue}40745[40745] {pull}41506[41506]
+
 *Elastic Log Driver*
 *Elastic Logging Plugin*
 
@@ -432,6 +461,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Filebeat*
 
+- Removed `bucket_timeout` config option for GCS input and replaced bucket context with parent program context. {issue}41107[41107] {pull}41970[41970]
 
 *Heartbeat*
 

Makefile

@@ -1,6 +1,6 @@
 BUILD_DIR=$(CURDIR)/build
 COVERAGE_DIR=$(BUILD_DIR)/coverage
-BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat x-pack/agentbeat x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/functionbeat x-pack/heartbeat x-pack/metricbeat x-pack/osquerybeat x-pack/packetbeat x-pack/winlogbeat
+BEATS?=auditbeat filebeat heartbeat metricbeat packetbeat winlogbeat x-pack/agentbeat x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/osquerybeat x-pack/packetbeat x-pack/winlogbeat
 PROJECTS=libbeat x-pack/libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env