Merge branch 'master' into feature/use-arch-in-the-package-binareis

.ci/jobs/beats.yml

@@ -17,7 +17,7 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|6\.[89]|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|6\.[89]|7\.16|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
         repo: 'beats'

.ci/schedule-daily.groovy

@@ -21,8 +21,11 @@ pipeline {
     stage('Nighly beats builds') {
       steps {
         runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
-        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.15')
+        // This should be `current_8` bump.getCurrentMinorReleaseFor8
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/8.0')
+        // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
+        // `next_minor_7`  bump.getNextMinorReleaseFor7
+        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.16')
       }
     }
   }

.ci/schedule-weekly.groovy

@@ -21,8 +21,11 @@ pipeline {
     stage('Weekly beats builds') {
       steps {
         runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.16')
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.15')
+        // This should be `current_8` bump.getCurrentMinorReleaseFor8
+        runBuild(quietPeriod: 1000, job: 'Beats/beats/8.0')
+        // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
+        // `next_minor_7`  bump.getNextMinorReleaseFor7
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
       }
     }
   }

.mergify.yml

@@ -101,10 +101,24 @@ pull_request_rules:
         - files~=^\.mergify\.yml$
     actions:
       delete_head_branch:
+  - name: notify the backport has not been merged yet
+    conditions:
+      - -merged
+      - -closed
+      - author=mergify[bot]
+      - "#check-success>0"
+      - schedule=Mon-Mon 06:00-10:00[Europe/Paris]
+      - "#assignee>=1"
+    actions:
+      comment:
+        message: |
+          This pull request has not been merged yet. Could you please review and merge it @{{ assignee | join(', @') }}? 🙏
   - name: notify the backport policy
     conditions:
       - -label~=^backport
       - base=master
+      - -merged
+      - -closed
     actions:
       comment:
         message: |
@@ -120,6 +134,8 @@ pull_request_rules:
   - name: remove-backport label
     conditions:
       - label~=backport-v
+      - -merged
+      - -closed
     actions:
       label:
         remove:

CHANGELOG.asciidoc

@@ -3,6 +3,11 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.0.0-beta1]]
+=== Beats version 8.0.0-beta1
+
+Changes will be described in a later RC / GA.
+
 [[release-notes-8.0.0-alpha2]]
 === Beats version 8.0.0-alpha2
 
@@ -12,6 +17,37 @@ Changes will be described in a later alpha / beta.
 === Beats version 8.0.0-alpha1
 
 Changes will be described in a later alpha / beta.
+[[release-notes-7.15.2]]
+=== Beats version 7.15.2
+https://github.com/elastic/beats/compare/v7.15.1...v7.15.2[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Beats dashboards use custom index when `setup.dashboards.index` is set. {issue}21232[21232] {pull}27901[27901]
+- Fix handling of float data types within processors. {issue}28279[28279] {pull}28280[28280]
+- Allow `clone3` syscall in seccomp filters. {pull}28117[28117]
+- Remove unnecessary escaping step in dashboard loading, so they can be displayed in Kibana. {pull}28395[28395]
+- Fix AWS proxy_url config from url to string type.  {pull}28725[28725]
+- Fix `fingerprint` processor to give it access to the `@timestamp` field. {issue}28683[28683]
+
+*Filebeat*
+
+- Fix initialization of http client in Cloudfoundry input. {issue}28271[28271] {pull}28277[28277]
+- Fix aws-s3 input by checking if GetObject API call response content type exists. {pull}28457[28457]
+- Set `url` as a pointer in the `httpjson` template context to ensure access to all methods. {pull}28695[28695]
+- Fix `google_workspace` documentation links. {pull}28657[28657]
+
+*Metricbeat*
+
+- Divide RDS metric cpu.total.pct by 100. {pull}28456[28456]
+
+*Packetbeat*
+
+- Handle truncated DNS records more gracefully. {issue}21495[21495] {pull}28297[28297]
+- Fix data stream name for network flows when running under Elastic Agent and Fleet. {pull}28408[28408]
+
 [[release-notes-7.15.1]]
 === Beats version 7.15.1
 https://github.com/elastic/beats/compare/v7.15.0...v7.15.1[View commits]

CHANGELOG.next.asciidoc

@@ -28,6 +28,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - add_docker_metadata processor: Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
 - Index template's default_fields setting is only populated with ECS fields. {pull}28596[28596] {issue}28215[28215]
 - tar.gz packages for ARM64 will now use the suffix `aarch64` rather than `arm64`. {pull}28813[28813]
+- Remove deprecated `--template` and `--ilm-policy` flags. Use `--index-management` instead. {pull}28870[28870]
+- Remove options `logging.files.suffix` and default to datetime endings. {pull}28927[28927]
 
 *Auditbeat*
 
@@ -38,6 +40,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix handling of long file names on Windows. {issue}25334[25334] {pull}28517[28517]
 - System/socket dataset: Fix uninstallation of return kprobes. {issue}28608[28608] {pull}28609[28609]
 - Replace usage of deprecated `process.ppid` field with `process.parent.pid`. {pull}28620[28620]
+- Fix auditbeat tracing struct decoding. {pull}28580[28580]
 
 *Filebeat*
 
@@ -62,8 +65,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - All modules: Replace usages of deprecated ECS fields `process.ppid` and `log.original` with `process.parent.pid` and `event.original`. {pull}28620[28620]
 - Replace usages of `host.user.*` fields with `user.*` in `cisco`, `microsoft` and `oracle` modules. {pull}28620[28620]
 - Remove `docker` input. Please use `filestream` input with `container` parser or `container` input. {pull}28817[28817]
+- Change `threatintel` module to use new `threat.*` ECS fields. {pull}29014[29014]
+- `filestream` and `log` inputs accept null (`\u0000`) as line terminator. {pull}28998[28998]
 
 *Heartbeat*
+- Change behavior in case of duplicate monitor IDs in configs to be last monitor wins. {pull}29041[29041]
 
 *Journalbeat*
 
@@ -135,15 +141,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Output errors when Kibana index pattern setup fails. {pull}20121[20121]
 - Fix issue in autodiscover that kept inputs stopped after config updates. {pull}20305[20305]
 - Add service resource in k8s cluster role. {pull}20546[20546]
-- Fixed documentation for commands in beats dev guide {pull}22194[22194]
-- Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss`
-- Beats dashboards use custom index when `setup.dashboards.index` is set. {issue}21232[21232] {pull}27901[27901]
-- Fix handling of float data types within processors. {issue}28279[28279] {pull}28280[28280]
-- Allow `clone3` syscall in seccomp filters. {pull}28117[28117]
-- Remove unnecessary escaping step in dashboard loading, so they can be displayed in Kibana. {pull}28395[28395]
+- Periodic metrics in logs will now report `libbeat.output.events.active` and `beat.memstats.rss` as gauges (rather than counters). {pull}22877[22877]
+- Fix discovery of Nomad allocations with multiple events during startup. {pull}28700[28700]
 - Allows disable pod events enrichment with deployment name {pull}28521[28521]
-- Fix AWS proxy_url config from url to string type.  {pull}28725[28725]
 - Fix `fingerprint` processor to give it access to the `@timestamp` field. {issue}28683[28683]
+- Fix the wrong beat name on monitoring and state endpoint {issue}27755[27755]
 
 *Auditbeat*
 
@@ -185,17 +187,20 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for username in cisco asa security negotiation logs {pull}26975[26975]
 - Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
 - Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352]
-- Fix initialization of http client in Cloudfoundry input. {issue}28271[28271] {pull}28277[28277]
-- Fix aws-s3 input by checking if GetObject API call response content type exists. {pull}28457[28457]
-- Set `url` as a pointer in the `httpjson` template context to ensure access to all methods. {pull}28695[28695]
-- Fix `google_workspace` documentation links. {pull}28657[28657]
+- Fix parsing of apache log levels including numbers. {pull}28717[28717]
+- Upgrade azure-eventhub sdk reference, contains potential checkpoint fixes. {pull}28919[28919]
+- Revert usageDetails api version to 2019-01-01. {pull}28995[28995]
+- Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
+- Fix `threatintel.misp` filters configuration. {issue}27970[27970]
 
 *Heartbeat*
 
 - Fix broken seccomp filtering and improve security via `setcap` and `setuid` when running as root on linux in containers. {pull}27878[27878]
 - Log browser `zip_url` download failures as `warn` instead of as `info`. {pull}28440[28440]
 - Properly locate base stream in fleet configs. {pull}28455[28455]
 - Stop logging params values. {pull}28774[28774]
+- Remove accidentally included cups library in docker images. {pull}28853[pull]
+- Fix broken monitors with newer versions of image relying on dup3. {pull}28938[pull]
 
 *Journalbeat*
 
@@ -231,19 +236,19 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
 - `beat` module respects `basepath` config option. {pull}28162[28162]
 - Fix list_docker.go {pull}28374[28374]
-- Divide RDS metric cpu.total.pct by 100. {pull}28456[28456]
 - Use xpack.enabled on SM modules to write into .monitoring indices when using Metricbeat standalone {pull}28365[28365]
+- Fix in rename processor to ingest metrics for `write.iops` to proper field instead of `write_iops` in rds metricset. {pull}28960[28960]
 
 *Packetbeat*
 
-- Handle truncated DNS records more gracefully. {issue}21495[21495] {pull}28297[28297]
-- Fix data stream name for network flows when running under Elastic Agent and Fleet. {pull}28408[28408]
 
 *Winlogbeat*
 
 - Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
 - Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
 - Add ECS 1.9 new users fields {pull}26509[26509]
+- Don't split hyphenated tokens {pull}28483[28483]
+- Correctly handle AccessMask if it is an integer or list of masks. {pull}29016[29016]
 
 *Functionbeat*
 
@@ -283,6 +288,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update kubernetes scheduler and controllermanager endpoints in elastic-agent-standalone-kubernetes.yaml with secure ports {pull}28675[28675]
 - Add options to configure k8s client qps/burst. {pull}28151[28151]
 - Update to ECS 8.0 fields. {pull}28620[28620]
+- Add http.pprof.enabled option to libbeat to allow http/pprof endpoints on the socket that libbeat creates for metrics. {issue}21965[21965]
+- Support custom analyzers in fields.yml. {issue}28540[28540] {pull}28926[28926]
 
 *Auditbeat*
 
@@ -336,6 +343,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update `aws-s3` input to connect to non AWS S3 buckets {issue}28222[28222] {pull}28234[28234]
 - Sophos UTM: Support logs containing hostname in syslog header. {pull}28638[28638]
 - Moving Oracle Filebeat module to GA. {pull}28754[28754]
+- Add support for '/var/log/pods/' path for add_kubernetes_metadata processor with `resource_type: pod`. {pull}28868[28868]
+- Add documentation for add_kubernetes_metadata processors `log_path` matcher. {pull}28868[28868]
+- Add support in aws-s3 input for s3 notification from SNS to SQS. {pull}28800[28800]
+- Add support in aws-s3 input for custom script parsing of s3 notifications. {pull}28946[28946]
+- Improve error handling in aws-s3 input for malformed s3 notifications. {issue}28828[28828] {pull}28946[28946]
 
 *Heartbeat*
 
@@ -360,6 +372,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added a new beta `enterprisesearch` module for Elastic Enterprise Search {pull}27549[27549]
 - Preliminary AIX support {pull}27954[27954]
 - Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
+- Update reference to gosigar pacakge for filesystem windows fix. {pull}28909[28909]
 
 *Packetbeat*
 

Jenkinsfile

@@ -836,7 +836,7 @@ def archiveTestOutput(Map args = [:]) {
         def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method
         def files = findFiles(glob: "${fileName}")
 
-        if (files?.length() > 0) {
+        if (files?.length > 0) {
           googleStorageUploadExt(
             bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
             credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",

NOTICE.txt

@@ -832,11 +832,11 @@ Contents of probable licence file $GOMODCACHE/code.cloudfoundry.org/go-loggregat
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-event-hubs-go/v3
-Version: v3.1.2
+Version: v3.3.15
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-event-hubs-go/v3@v3.1.2/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-event-hubs-go/v3@v3.3.15/LICENSE:
 
     MIT License
 
@@ -863,15 +863,15 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-event-hubs
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-sdk-for-go
-Version: v57.0.0+incompatible
+Version: v59.0.0+incompatible
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go@v57.0.0+incompatible/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-sdk-for-go@v59.0.0+incompatible/LICENSE.txt:
 
 The MIT License (MIT)
 
-Copyright (c) 2021 Microsoft
+Copyright (c) Microsoft Corporation.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -891,6 +891,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-storage-blob-go
 Version: v0.8.0
@@ -8276,11 +8277,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected]
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/gosigar
-Version: v0.14.1
+Version: v0.14.2
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].2/LICENSE:
 
                               Apache License
                         Version 2.0, January 2004
@@ -20250,11 +20251,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/azure-amqp-common-go/v3
-Version: v3.0.0
+Version: v3.2.1
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-amqp-common-go/v3@v3.0.0/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-amqp-common-go/v3@v3.2.1/LICENSE:
 
     MIT License
 
@@ -20311,15 +20312,16 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/azure-pipeline-g
 
 --------------------------------------------------------------------------------
 Dependency : github.com/Azure/go-amqp
-Version: v0.12.6
+Version: v0.16.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.12.6/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/!azure/go-amqp@v0.16.0/LICENSE:
 
     MIT License
 
-    Copyright (c) Microsoft Corporation.
+    Copyright (C) 2017 Kale Blankenship
+    Portions Copyright (C) Microsoft Corporation
 
     Permission is hereby granted, free of charge, to any person obtaining a copy
     of this software and associated documentation files (the "Software"), to deal

README.md

@@ -74,14 +74,6 @@ create your own Beat.
 
 Please start by reading our [CONTRIBUTING](CONTRIBUTING.md) file.
 
-If you are creating a new Beat, you don't need to submit the code to this
-repository. You can simply start working in a new repository and make use of the
-libbeat packages, by following our [developer
-guide](https://www.elastic.co/guide/en/beats/libbeat/current/new-beat.html).
-After you have a working prototype, open a pull request to add your Beat to the
-list of [community
-Beats](https://github.com/elastic/beats/blob/master/libbeat/docs/communitybeats.asciidoc).
-
 ## Building Beats from the Source
 
 See our [CONTRIBUTING](CONTRIBUTING.md) file for information about setting up

auditbeat/auditbeat.reference.yml

@@ -1443,11 +1443,6 @@ logging.files:
   # file. Defaults to true.
   # rotateonstartup: true
 
-  # Rotated files are either suffixed with a number e.g. auditbeat.1 when
-  # renamed during rotation. Or when set to date, the date is added to
-  # the end of the file. On rotation a new file is created, older files are untouched.
-  #suffix: count
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
@@ -1632,6 +1627,10 @@ logging.files:
 # `http.user`.
 #http.named_pipe.security_descriptor:
 
+# Defines if the HTTP pprof endpoints are enabled.
+# It is recommended that this is only enabled on localhost as these endpoints may leak data.
+#http.pprof.enabled: false
+
 # ============================== Process Security ==============================
 
 # Enable or disable seccomp system call filtering on Linux. Default is enabled.