Merge branch 'master' into ga_zoom

elastic · Sep 30, 2021 · 9727e4e · 9727e4e
2 parents 7f04bd0 + 4b1f699
commit 9727e4e
Show file tree

Hide file tree

Showing 171 changed files with 1,375 additions and 642 deletions.
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -40,8 +40,6 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
-    booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
-    booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
   }
   stages {
     stage('Filter build') {
@@ -125,12 +123,6 @@ pipeline {
               stage('Package Linux'){
                 agent { label 'ubuntu-18 && immutable' }
                 options { skipDefaultCheckout() }
-                when {
-                  beforeAgent true
-                  expression {
-                    return params.linux
-                  }
-                }
                 environment {
                   HOME = "${env.WORKSPACE}"
                   PLATFORMS = [
@@ -224,12 +216,6 @@ pipeline {
               stage('Package Docker images for linux/arm64'){
                 agent { label 'arm' }
                 options { skipDefaultCheckout() }
-                when {
-                  beforeAgent true
-                  expression {
-                    return params.arm
-                  }
-                }
                 environment {
                   HOME = "${env.WORKSPACE}"
                   PACKAGES = "docker"
@@ -335,7 +321,7 @@ def tagAndPush(Map args = [:]) {
   }
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
-  // 
+  //
   if(beatName == 'elastic-agent'){
       variants.add("-complete")
   }

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -95,7 +95,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Journalbeat*
 
 - Improve parsing of syslog.pid in journalbeat to strip the username when present {pull}16116[16116]
-
+- Rename field `journald.process.capabilites` to `journald.process.capabilities` to fix spelling. {pull}28065[28065]
+- Rename field `log.syslog.facility.name` to `log.syslog.facility.code` because the value is numeric rather than the facility name. {pull}28065[28065]
 
 *Metricbeat*
 
@@ -405,6 +406,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix remote_write flaky test. {pull}21173[21173]
 - Remove io.time from windows {pull}22237[22237]
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
+- `beat` module respects `basepath` config option. {pull}28162[28162]
 
 *Packetbeat*
 
@@ -503,6 +505,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Kafka is now supported up to version 2.8.0. {pull}27720[27720]
 - Add Huawei Cloud provider to add_cloud_metadata. {pull}27607[27607]
 - Add default seccomp policy for linux arm64. {pull}27955[27955]
+- Add cluster level add_kubernetes_metadata support for centralized enrichment {pull}24621[24621]
 
 *Auditbeat*
 
@@ -726,11 +729,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937]
 - Add Base64 encoded HMAC and UUID template functions to `httpjson` input {pull}27873[27873]
 - Release checkpoint module as GA. {pull}27814[27814]
+- Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 - Release zoom module as GA. {pull}28106[28106]
 
 *Heartbeat*
 
 - Add mime type detection for http responses. {pull}22976[22976]
+- Support JSON expressions / validation of JSON arrays. {pull}28073[28073]
 
 *Journalbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -806,7 +806,10 @@ def archiveTestOutput(Map args = [:]) {
         }
         def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method
         dir("${BASE_DIR}"){
-          cmd(label: "List files to upload", script: "ls -l ${BASE_DIR}/${fileName}")
+          def files = findFiles(glob: "${fileName}")
+          files.each { file ->
+            echo "${file.name}"
+          }
           googleStorageUploadExt(
             bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
             credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -2181,6 +2181,48 @@ SOFTWARE.
 
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/PaesslerAG/gval
+Version: v1.0.0
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!paessler!a!g/[email protected]/LICENSE:
+
+Copyright (c) 2017, Paessler AG <[email protected]>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+--------------------------------------------------------------------------------
+Dependency : github.com/PaesslerAG/jsonpath
+Version: v0.1.1
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/!paessler!a!g/[email protected]/LICENSE:
+
+Copyright (c) 2017, Paessler AG <[email protected]>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/sarama
 Version: v1.19.1-0.20210823122811-11c3ef800752

diff --git a/dev-tools/mage/fields.go b/dev-tools/mage/fields.go
@@ -118,6 +118,11 @@ func GenerateAllInOneFieldsGo() error {
 	return GenerateFieldsGo("fields.yml", "include/fields.go")
 }
 
+// GenerateMetricbeatAllInOneFieldsGo generates an all-in-one fields.go file for metricbeat.
+func GenerateMetricbeatAllInOneFieldsGo() error {
+	return GenerateFieldsGo("fields.yml", "include/fields/fields.go")
+}
+
 // GenerateFieldsGo generates a .go file containing the fields.yml data.
 func GenerateFieldsGo(fieldsYML, out string) error {
 	const assetCmdPath = "dev-tools/cmd/asset/asset.go"

diff --git a/filebeat/docs/inputs/input-journald.asciidoc b/filebeat/docs/inputs/input-journald.asciidoc
@@ -7,6 +7,8 @@
 <titleabbrev>journald</titleabbrev>
 ++++
 
+experimental[]
+
 https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html[`journald`]
 is a system service that collects and stores logging data. The `journald` input
 reads this log data and the metadata associated with it.
@@ -50,7 +52,7 @@ possible.
     - _TRANSPORT=kernel
   processors:
     - drop_event:
-        when.not.regex.message: '^iptables'
+        when.not.regexp.message: '^iptables'
 ----
 
 Each example adds the `id` for the input to ensure the cursor is persisted to

diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc
@@ -160,7 +160,10 @@ Default to be 300 seconds.
 
 *`var.api_timeout`*::
 
-Maximum duration before AWS API request will be interrupted. Default to be 120 seconds.
+The maximum duration of the AWS API call. If it exceeds the timeout, the AWS API
+call will be interrupted. The default AWS API timeout is `120s`.
+
+The API timeout must be longer than the `sqs.wait_time` value.
 
 *`var.bucket_arn`*::
 

diff --git a/filebeat/docs/modules/haproxy.asciidoc b/filebeat/docs/modules/haproxy.asciidoc
@@ -6,7 +6,7 @@ This file is generated! See scripts/docs_collector.py
 :modulename: haproxy
 :has-dashboards: true
 
-== haproxy module
+== HAproxy module
 
 include::{libbeat-dir}/shared/integration-link.asciidoc[]
 

diff --git a/filebeat/docs/modules/mysqlenterprise.asciidoc b/filebeat/docs/modules/mysqlenterprise.asciidoc
@@ -13,8 +13,6 @@ This file is generated! See scripts/docs_collector.py
 
 beta[]
 
-include::{libbeat-dir}/shared/integration-link.asciidoc[]
-
 This is a module for different types of MySQL logs. Currently focusing on data from the MySQL Enterprise Audit Plugin in JSON format.
 
 To configure the the Enterprise Audit Plugin to output in JSON format please follow the directions in the https://dev.mysql.com/doc/refman/8.0/en/audit-log-file-formats.html[MySQL Documentation.]

diff --git a/filebeat/docs/modules/nats.asciidoc b/filebeat/docs/modules/nats.asciidoc
@@ -6,11 +6,11 @@ This file is generated! See scripts/docs_collector.py
 :modulename: nats
 :has-dashboards: false
 
-== nats module
+== NATS module
 
 include::{libbeat-dir}/shared/integration-link.asciidoc[]
 
-This is the nats module.
+This is the NATS module.
 
 include::../include/what-happens.asciidoc[]
 
@@ -37,7 +37,7 @@ include::../include/var-paths.asciidoc[]
 [float]
 === Dashboard
 
-The Nats module comes with a predefined dashboard. For example:
+The +{modulename}+ module comes with a predefined dashboard. For example:
 
 image::./images/filebeat_nats_dashboard.png[]
 

diff --git a/filebeat/module/haproxy/_meta/docs.asciidoc b/filebeat/module/haproxy/_meta/docs.asciidoc
@@ -1,7 +1,7 @@
 :modulename: haproxy
 :has-dashboards: true
 
-== haproxy module
+== HAproxy module
 
 include::{libbeat-dir}/shared/integration-link.asciidoc[]
 

diff --git a/filebeat/module/nats/_meta/docs.asciidoc b/filebeat/module/nats/_meta/docs.asciidoc
@@ -1,11 +1,11 @@
 :modulename: nats
 :has-dashboards: false
 
-== nats module
+== NATS module
 
 include::{libbeat-dir}/shared/integration-link.asciidoc[]
 
-This is the nats module.
+This is the NATS module.
 
 include::../include/what-happens.asciidoc[]
 
@@ -32,7 +32,7 @@ include::../include/var-paths.asciidoc[]
 [float]
 === Dashboard
 
-The Nats module comes with a predefined dashboard. For example:
+The +{modulename}+ module comes with a predefined dashboard. For example:
 
 image::./images/filebeat_nats_dashboard.png[]
 

diff --git a/go.mod b/go.mod
@@ -20,6 +20,8 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0
 	github.com/Masterminds/semver v1.4.2
 	github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5
+	github.com/PaesslerAG/gval v1.0.0
+	github.com/PaesslerAG/jsonpath v0.1.1
 	github.com/Shopify/sarama v1.27.0
 	github.com/StackExchange/wmi v0.0.0-20170221213301-9f32b5905fd6
 	github.com/aerospike/aerospike-client-go v1.27.1-0.20170612174108-0f3b54da6bdc

diff --git a/go.sum b/go.sum
@@ -87,6 +87,11 @@ github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEY
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PaesslerAG/gval v1.0.0 h1:GEKnRwkWDdf9dOmKcNrar9EA1bz1z9DqPIO1+iLzhd8=
+github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I=
+github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
+github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk=
+github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
 github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc=

diff --git a/heartbeat/_meta/config/beat.reference.yml.tmpl b/heartbeat/_meta/config/beat.reference.yml.tmpl
@@ -223,7 +223,12 @@ heartbeat.monitors:
     # Required response contents.
     #body:
 
-    # Parses the body as JSON, then checks against the given condition expression
+    # Parses the body as JSON, then checks against the given expression
+    #json:
+    #- description: Explanation of what the check does
+-   #  expression: 'myField == "expectedValue"'
+
+    # (Deprecated: see 'expression' above) Parses the body as JSON, then checks against the given condition expression
     #json:
     #- description: Explanation of what the check does
     #  condition:

diff --git a/heartbeat/docs/monitors/monitor-http.asciidoc b/heartbeat/docs/monitors/monitor-http.asciidoc
@@ -258,12 +258,6 @@ This monitor examines match successfully only when 'foo' or 'Foo' in body AND no
   name: Demo Service
   schedule: '@every 5s'
   urls: ["http://localhost:8080/demo/add"]
-  check.request:
-    method: POST
-    headers:
-      'Content-Type': 'application/x-www-form-urlencoded'
-    # urlencode the body:
-    body: "name=first&email=someemail%40someemailprovider.com"
   check.response:
     status: [200, 201]
     body:
@@ -275,11 +269,32 @@ This monitor examines match successfully only when 'foo' or 'Foo' in body AND no
         - Bar
 -------------------------------------------------------------------------------
 
-*`json`*:: A list of <<conditions,condition>> expressions executed against the body when parsed as JSON. Body sizes
+*`json`*:: A list of expressions or <<conditions,condition>> statements (now deprecated) executed against the body when parsed as JSON. Body sizes
 must be less than or equal to 100 MiB.
 
-The following configuration shows how to check the response when the body
-contains JSON:
+The following configuration shows how to check the response using [gval](https://github.com/PaesslerAG/gval/blob/master/README.md) expressions when the body contains JSON:
+
+[source,yaml]
+-------------------------------------------------------------------------------
+- type: http
+  id: demo-service
+  name: Demo Service
+  schedule: '@every 5s'
+  hosts: ["https://localhost:9200/_/nodes/stats"]
+  username: elastic
+  password: changeme
+  check.response:
+    status: [200]
+    json:
+      - description: check status
+        expression: 'foo.bar == "myValue"'
+-------------------------------------------------------------------------------
+
+Expressions can be much more complex than simple equality. They can also use [jsonpath](https://goessner.net/articles/JsonPath/) syntax. Note that strings must be double quoted with `"` rather than single quoted with `'` in the `gval` variant of jsonpath. Please note that jsonpath sub-expressions must start with `$.`, for instance `'$.nodes[?(@.name=="myname")] != []'` will check that the `nodes` map has at least one value with the name 'myname'.
+
+When working with responses that are returned in the form of a JSON array at the root rather than an object jsonpath can be used as well. As an example `$.[0].foo == "bar"` tests that the first item in the response has an attribute `foo` that has the value "bar".
+
+JSON bodies can also be checked via the now deprecated `condition` option, which is not as powerful as `expression`. The following configuration shows how to check the response using a `condition` statement when the body contains JSON:
 
 [source,yaml]
 -------------------------------------------------------------------------------

diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
@@ -223,7 +223,12 @@ heartbeat.monitors:
     # Required response contents.
     #body:
 
-    # Parses the body as JSON, then checks against the given condition expression
+    # Parses the body as JSON, then checks against the given expression
+    #json:
+    #- description: Explanation of what the check does
+-   #  expression: 'myField == "expectedValue"'
+
+    # (Deprecated: see 'expression' above) Parses the body as JSON, then checks against the given condition expression
     #json:
     #- description: Explanation of what the check does
     #  condition:
-Original file line number
+Diff line change
@@ Expand Up / @@ -13,8 +13,6 @@ This file is generated! See scripts/docs_collector.py @@
     beta[]
-    include::{libbeat-dir}/shared/integration-link.asciidoc[]
     This is a module for different types of MySQL logs. Currently focusing on data from the MySQL Enterprise Audit Plugin in JSON format.
     To configure the the Enterprise Audit Plugin to output in JSON format please follow the directions in the https://dev.mysql.com/doc/refman/8.0/en/audit-log-file-formats.html[MySQL Documentation.]
@@ Expand Down @@