Merge branch 'master' into archive-system-tests

* master: (25 commits)
  Fix UBI source URL (elastic#26384)
  Skip test_rotating_file in osx and windows (elastic#26379)
  Remove outdated k8s manifests for managed elastic-agent (elastic#26368)
  Enable agent to send custom headers to kibana/ES (elastic#26275)
  [Automation] Update elastic stack version to 8.0.0-943ef2c0 for testing (elastic#26354)
  Make the Syslog input GA (elastic#26293)
  Move Kerberos FAST config flag to shared kerberos config (elastic#26141)
  Add k8s cluster identifiers (elastic#26056)
  Store message from MongoDB json logs in message field (elastic#26338)
  update threatintel ECS version (elastic#26274)
  update envoyproxy ECS version (elastic#26277)
  [Filebeat] [MongoDB] Support MongoDB 4.4 json logs (elastic#24774)
  Update go-structform to 0.0.9 (elastic#26251)
  Forward port 7.13.2 changelog to master (elastic#26323)
  Updated filter expression for filtering 86 artifacts (elastic#26313)
  Osquerybeat: Align with the rest of the beats, set the ECS version (elastic#26324)
  [Packetbeat] Add `url.extension` to Packetbeat HTTP events (elastic#25999)
  Change link to snapshots in README (elastic#26317)
  Don't include full ES index template in errors (elastic#25743)
  First refactor of the system module - system/cpu and system/core (elastic#25771)
  ...

.go-version

@@ -1 +1 @@
-1.16.4
+1.16.5

CHANGELOG-developer.next.asciidoc

@@ -116,3 +116,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.12. {pull}25629[25629]
 - Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
 - Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]
+- Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]

CHANGELOG.asciidoc

@@ -3,6 +3,31 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.13.2]]
+=== Beats version 7.13.2
+https://github.com/elastic/beats/compare/v7.13.1...v7.13.2[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix ILM alias creation when write alias exists and initial index does not exist. {pull}26146[26146]
+- Fix ILM setup log reporting that a policy or an alias was created, even though the creation of any resource was disabled. {issue}24046[24046] {pull}24480[24480]
+- Fix ILM alias not being created if `setup.ilm.check_exists: false` and `setup.ilm.overwrite: true` has been configured. {pull}24480[24480]
+- Allow cgroup self-monitoring to see alternate `hostfs` paths. {pull}24334[24334]
+- Fix `make setup` instructions for a new Beat. {pull}24944[24944]
+- Fix out-of-date FreeBSD vagrantbox. {pull}25652[25652]
+- Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
+- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally. {pull}26176[26176]
+
+*Auditbeat*
+
+- auditd: Fix kernel deadlock when netlink congestion causes "no buffer space available" errors. {issue}26031[26031] {pull}26032[26032]
+
+*Filebeat*
+
+- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
+
 [[release-notes-7.13.1]]
 === Beats version 7.13.1
 https://github.com/elastic/beats/compare/v7.13.0...v7.13.1[View commits]

CHANGELOG.next.asciidoc

@@ -217,7 +217,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915]
 - Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
 - Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
-- The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21259[21258]
 - Orderly close processors when processing pipelines are not needed anymore to release their resources. {pull}16349[16349]
 - Fix memory leak and events duplication in docker autodiscover and add_docker_metadata. {pull}21851[21851]
 - Fixed documentation for commands in beats dev guide {pull}22194[22194]
@@ -233,15 +232,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add FAQ entry for madvdontneed variable {pull}23429[23429]
 - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
 - Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
-- Fix ILM setup log reporting that a policy or an alias was created, even though the creation of any resource was disabled. {issue}24046[24046] {pull}24480[24480]
-- Fix ILM alias not being created if `setup.ilm.check_exists: false` and `setup.ilm.overwrite: true` has been configured. {pull}24480[24480]
 - Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318]
-- Allow cgroup self-monitoring to see alternate `hostfs` paths {pull}24334[24334]
-- Fix 'make setup' instructions for a new beat {pull}24944[24944]
-- Fix out of date FreeBSD vagrantbox. {pull}25652[25652]
-- Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
 - Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143]
-- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally {pull}26176[26176]
+- Omit full index template from errors that occur while loading the template. {pull}25743[25743]
 - In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively.
 
 *Auditbeat*
@@ -264,7 +257,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
 - system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
 - Note incompatibility of system/socket on ARM. {pull}23381[23381]
-- auditd: Fix kernel deadlock when netlink congestion causes "no buffer space available" errors. {issue}26031[26031] {pull}26032[26032]
 
 *Filebeat*
 
@@ -282,6 +274,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
 - Fix default config template values for paths on oracle module: {pull}26276[26276]
+- Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
 
 *Filebeat*
 
@@ -390,7 +383,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
 - Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
-- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
 
 *Heartbeat*
 
@@ -498,6 +490,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
+- Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
 
 *Packetbeat*
 
@@ -595,6 +588,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow node/namespace metadata to be disabled on kubernetes metagen and ensure add_kubernetes_metadata honors host {pull}23012[23012]
 - Add support for defining explicitly named dynamic templates without path/type match criteria {pull}25422[25422]
 - Improve ES output error insights. {pull}25825[25825]
+- Add orchestrator.cluster.name/url fields as k8s metadata {pull}26056[26056]
 - Libbeat: report beat version to monitoring. {pull}26214[26214]
 
 *Auditbeat*
@@ -817,13 +811,18 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776]
 - Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
 - Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
+- Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774]
 - Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
 - Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168]
 - http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764]
 - Make `filestream` input GA. {pull}26127[26127]
 - Add new `parser` to `filestream` input: `container`. {pull}26115[26115]
 - Add support for ISO8601 timestamps in Zeek fileset {pull}25564[25564]
 - Add `preserve_original_event` option to `o365audit` input. {pull}26273[26273]
+- Add `log.flags` to events created by the `aws-s3` input. {pull}26267[26267]
+- Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
+- RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
+
 
 *Heartbeat*
 
@@ -974,6 +973,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for "http.request.mime_type" and "http.response.mime_type". {pull}22940[22940]
 - Upgrade to ECS 1.8.0. {pull}23783[23783]
 - Add `event.type: [connection]` to flow events and include `end` for final flows. {pull}24564[24564]
+- Add `url.extension` to HTTP events {issue}25990[25990] {pull}25999[25999]
 
 *Functionbeat*
 

NOTICE.txt

@@ -7696,11 +7696,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-seccomp-bpf@
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-structform
-Version: v0.0.8
+Version: v0.0.9
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].8/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/[email protected].9/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004

README.md

@@ -89,7 +89,7 @@ your dev environment to build Beats from the source.
 
 ## Snapshots
 
-For testing purposes, we generate snapshot builds that you can find [here](https://beats-ci.elastic.co/job/Beats/job/packaging/job/master/lastSuccessfulBuild/gcsObjects/). Please be aware that these are built on top of master and are not meant for production.
+For testing purposes, we generate snapshot builds that you can find [here](https://artifacts-api.elastic.co/v1/search/8.0-SNAPSHOT/). Please be aware that these are built on top of master and are not meant for production.
 
 ## CI
 

Vagrantfile

@@ -307,6 +307,7 @@ Vagrant.configure("2") do |config|
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: $freebsdShellUpdate, privileged: true
     c.vm.provision "shell", inline: gvmProvision(arch="amd64", os="freebsd"), privileged: false
+    c.vm.provision "shell", inline: "sudo mount -t linprocfs /dev/null /proc", privileged: false
   end
 
   # OpenBSD 6.0

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4
+FROM golang:1.16.5
 
 RUN \
     apt-get update \

auditbeat/auditbeat.reference.yml

@@ -594,6 +594,7 @@ output.elasticsearch:
   # Kerberos realm.
   #kerberos.realm: ELASTIC
 
+
 # ------------------------------ Logstash Output -------------------------------
 #output.logstash:
   # Boolean flag to enable or disable the output module.
@@ -852,10 +853,6 @@ output.elasticsearch:
   # purposes.  The default is "beats".
   #client_id: beats
 
-  # Enables Kerberos FAST authentication in the Kafka output. This may
-  # conflict with certain Active Directory configurations.
-  #enable_krb5_fast: false
-
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
 
@@ -934,6 +931,10 @@ output.elasticsearch:
   # Kerberos realm.
   #kerberos.realm: ELASTIC
 
+  # Enables Kerberos FAST authentication. This may
+  # conflict with certain Active Directory configurations.
+  #kerberos.enable_krb5_fast: false
+
 # -------------------------------- Redis Output --------------------------------
 #output.redis:
   # Boolean flag to enable or disable the output module.

deploy/kubernetes/Makefile

@@ -1,4 +1,4 @@
-ALL=filebeat metricbeat auditbeat heartbeat elastic-agent-standalone elastic-agent
+ALL=filebeat metricbeat auditbeat heartbeat elastic-agent-standalone
 BEAT_VERSION=$(shell head -n 1 ../../libbeat/docs/version.asciidoc | cut -c 17- )
 
 .PHONY: all $(ALL)

deploy/kubernetes/auditbeat-kubernetes.yaml

@@ -226,6 +226,34 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
 ---
 apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: auditbeat
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: auditbeat
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: auditbeat
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: auditbeat-kubeadm-config
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: auditbeat
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: auditbeat-kubeadm-config
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: auditbeat
@@ -243,6 +271,36 @@ rules:
     - replicasets
   verbs: ["get", "list", "watch"]
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: auditbeat
+  # should be the namespace where auditbeat is running
+  namespace: kube-system
+  labels:
+    k8s-app: auditbeat
+rules:
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs: ["get", "create", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: auditbeat-kubeadm-config
+  namespace: kube-system
+  labels:
+    k8s-app: auditbeat
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    resourceNames:
+      - kubeadm-config
+    verbs: ["get"]
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:

deploy/kubernetes/auditbeat/auditbeat-role-binding.yaml

@@ -10,3 +10,31 @@ roleRef:
   kind: ClusterRole
   name: auditbeat
   apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: auditbeat
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: auditbeat
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: auditbeat
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: auditbeat-kubeadm-config
+  namespace: kube-system
+subjects:
+  - kind: ServiceAccount
+    name: auditbeat
+    namespace: kube-system
+roleRef:
+  kind: Role
+  name: auditbeat-kubeadm-config
+  apiGroup: rbac.authorization.k8s.io

deploy/kubernetes/auditbeat/auditbeat-role.yaml

@@ -15,3 +15,33 @@ rules:
   resources:
     - replicasets
   verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: auditbeat
+  # should be the namespace where auditbeat is running
+  namespace: kube-system
+  labels:
+    k8s-app: auditbeat
+rules:
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs: ["get", "create", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: auditbeat-kubeadm-config
+  namespace: kube-system
+  labels:
+    k8s-app: auditbeat
+rules:
+  - apiGroups: [""]
+    resources:
+      - configmaps
+    resourceNames:
+      - kubeadm-config
+    verbs: ["get"]