Merge branch 'master' into cloudwatch_ga

.ci/packaging.groovy

@@ -40,8 +40,6 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
-    booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
-    booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
   }
   stages {
     stage('Filter build') {
@@ -125,12 +123,6 @@ pipeline {
               stage('Package Linux'){
                 agent { label 'ubuntu-18 && immutable' }
                 options { skipDefaultCheckout() }
-                when {
-                  beforeAgent true
-                  expression {
-                    return params.linux
-                  }
-                }
                 environment {
                   HOME = "${env.WORKSPACE}"
                   PLATFORMS = [
@@ -224,12 +216,6 @@ pipeline {
               stage('Package Docker images for linux/arm64'){
                 agent { label 'arm' }
                 options { skipDefaultCheckout() }
-                when {
-                  beforeAgent true
-                  expression {
-                    return params.arm
-                  }
-                }
                 environment {
                   HOME = "${env.WORKSPACE}"
                   PACKAGES = "docker"
@@ -335,7 +321,7 @@ def tagAndPush(Map args = [:]) {
   }
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
-  // 
+  //
   if(beatName == 'elastic-agent'){
       variants.add("-complete")
   }

CHANGELOG.next.asciidoc

@@ -95,7 +95,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Journalbeat*
 
 - Improve parsing of syslog.pid in journalbeat to strip the username when present {pull}16116[16116]
-
+- Rename field `journald.process.capabilites` to `journald.process.capabilities` to fix spelling. {pull}28065[28065]
+- Rename field `log.syslog.facility.name` to `log.syslog.facility.code` because the value is numeric rather than the facility name. {pull}28065[28065]
 
 *Metricbeat*
 
@@ -405,6 +406,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix remote_write flaky test. {pull}21173[21173]
 - Remove io.time from windows {pull}22237[22237]
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
+- `beat` module respects `basepath` config option. {pull}28162[28162]
 
 *Packetbeat*
 
@@ -728,6 +730,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Base64 encoded HMAC and UUID template functions to `httpjson` input {pull}27873[27873]
 - Release checkpoint module as GA. {pull}27814[27814]
 - Make aws-cloudwatch input GA. {pull}28161[28161]
+- Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 
 *Heartbeat*
 

filebeat/docs/inputs/input-journald.asciidoc

@@ -7,6 +7,8 @@
 <titleabbrev>journald</titleabbrev>
 ++++
 
+experimental[]
+
 https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html[`journald`]
 is a system service that collects and stores logging data. The `journald` input
 reads this log data and the metadata associated with it.
@@ -50,7 +52,7 @@ possible.
     - _TRANSPORT=kernel
   processors:
     - drop_event:
-        when.not.regex.message: '^iptables'
+        when.not.regexp.message: '^iptables'
 ----
 
 Each example adds the `id` for the input to ensure the cursor is persisted to

filebeat/docs/modules/aws.asciidoc

@@ -160,7 +160,10 @@ Default to be 300 seconds.
 
 *`var.api_timeout`*::
 
-Maximum duration before AWS API request will be interrupted. Default to be 120 seconds.
+The maximum duration of the AWS API call. If it exceeds the timeout, the AWS API
+call will be interrupted. The default AWS API timeout is `120s`.
+
+The API timeout must be longer than the `sqs.wait_time` value.
 
 *`var.bucket_arn`*::
 

journalbeat/_meta/fields.common.yml

@@ -212,10 +212,10 @@
               example: 1
               description: >
                 The ID of the user which runs the process.
-            - name: capabilites
+            - name: capabilities
               required: false
               description: >
-                The effective capabilites of the process.
+                The effective capabilities of the process.
     - name: systemd
       type: group
       description: >

journalbeat/docs/fields.asciidoc

@@ -626,10 +626,10 @@ required: False
 
 --
 
-*`journald.process.capabilites`*::
+*`journald.process.capabilities`*::
 +
 --
-The effective capabilites of the process.
+The effective capabilities of the process.
 
 
 required: False

journalbeat/pkg/journalfield/default.go

@@ -48,7 +48,7 @@ var journaldEventFields = FieldConversion{
 	sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID:    integer("journald.audit.login_uid"),
 	sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION:     text("journald.audit.session"),
 	sdjournal.SD_JOURNAL_FIELD_BOOT_ID:           text("journald.host.boot_id"),
-	sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE:     text("journald.process.capabilites"),
+	sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE:     text("journald.process.capabilities"),
 	sdjournal.SD_JOURNAL_FIELD_CMDLINE:           text("journald.process.command_line"),
 	sdjournal.SD_JOURNAL_FIELD_CODE_FILE:         text("journald.code.file"),
 	sdjournal.SD_JOURNAL_FIELD_CODE_FUNC:         text("journald.code.func"),
@@ -61,7 +61,7 @@ var journaldEventFields = FieldConversion{
 	sdjournal.SD_JOURNAL_FIELD_MESSAGE:           text("message"),
 	sdjournal.SD_JOURNAL_FIELD_PID:               integer("journald.pid"),
 	sdjournal.SD_JOURNAL_FIELD_PRIORITY:          integer("syslog.priority", "log.syslog.priority"),
-	sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY:   integer("syslog.facility", "log.syslog.facility.name"),
+	sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY:   integer("syslog.facility", "log.syslog.facility.code"),
 	sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: text("syslog.identifier"),
 	sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID:        integer("syslog.pid"),
 	sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP:    text("systemd.cgroup"),

libbeat/outputs/logstash/docs/logstash.asciidoc

@@ -282,7 +282,7 @@ are sticky, operating behind load balancers can lead to uneven load distribution
 Specifying a TTL on the connection allows to achieve equal connection distribution between the
 instances.  Specifying a TTL of 0 will disable this feature.
 
-The default value is 0.
+The default value is 0. This setting accepts {beats-ref}/config-file-format-type.html#_duration[duration] data type values.
 
 NOTE: The "ttl" option is not yet supported on an async {ls} client (one with the "pipelining" option set).
 

libbeat/tests/system/beat/beat.py

@@ -741,14 +741,18 @@ def assert_explicit_ecs_version_set(self, module, fileset):
         Assert that the module explicitly sets the ECS version field.
         """
         def get_config_paths(modules_path, module, fileset):
-            pathname = os.path.abspath(modules_path +
-                                       "/" +
-                                       module +
-                                       "/" +
-                                       fileset +
-                                       "/" +
-                                       "config/*.yml")
-            return glob.glob(pathname)
+            fileset_path = os.path.abspath(modules_path +
+                                           "/" +
+                                           module +
+                                           "/" +
+                                           fileset +
+                                           "/")
+            paths = []
+            for x in ["config/*.yml", "ingest/*.yml", "ingest/*.json"]:
+                pathname = os.path.join(fileset_path, x)
+                paths.extend(glob.glob(pathname))
+
+            return paths
 
         def is_ecs_version_set(path):
             # parsing the yml file would be better but go templates in
@@ -759,9 +763,7 @@ def is_ecs_version_set(path):
                         return True
             return False
 
-        errors = []
         for cfg_path in get_config_paths(self.modules_path, module, fileset):
-            if not is_ecs_version_set(cfg_path):
-                errors.append("{}".format(cfg_path))
-        if len(errors) > 0:
-            raise Exception("{}/{} ecs.version not explicitly set in:\n{}".format(module, fileset, '\n'.join(errors)))
+            if is_ecs_version_set(cfg_path):
+                return
+        raise Exception("{}/{} ecs.version not explicitly set in config or pipeline".format(module, fileset))

metricbeat/docs/modules/aws.asciidoc

@@ -238,7 +238,7 @@ applications and services.
 
 [float]
 [[aws-api-requests]]
-== AWS API requests count per metricset
+== AWS API requests count
 This session is to document what are the AWS API called made by each metricset
 in `aws` module. This will be useful for users to estimate costs for using `aws`
 module.
@@ -251,7 +251,6 @@ ListMetrics max page size: 500, based on https://docs.aws.amazon.com/AmazonCloud
 GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html[AWS API GetMetricData]
 
 [float]
-=== `cloudwatch`
 |===
 | AWS API Name | AWS API Count | Frequency
 | IAM ListAccountAliases | 1 | Once on startup
@@ -262,52 +261,6 @@ GetMetricData max page size: 100, based on https://docs.aws.amazon.com/AmazonClo
 |===
 `billing`, `ebs`, `elb`, `sns`, `usage` and `lambda` are the same as `cloudwatch` metricset.
 
-[float]
-=== `ec2`
-|===
-| AWS API Name | AWS API Count | Frequency
-| IAM ListAccountAliases | 1 | Once on startup
-| STS GetCallerIdentity | 1 | Once on startup
-| EC2 DescribeRegions| 1 | Once on startup
-| EC2 DescribeInstances | 1 | Per region per collection period
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
-| CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per collection period
-|===
-
-[float]
-=== `rds`
-|===
-| AWS API Name | AWS API Count | Frequency
-| IAM ListAccountAliases | 1 | Once on startup
-| STS GetCallerIdentity | 1 | Once on startup
-| EC2 DescribeRegions| 1 | Once on startup
-| RDS DescribeDBInstances | 1 | Per region per collection period
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
-| CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per collection period
-|===
-
-[float]
-=== `sqs`
-|===
-| AWS API Name | AWS API Count | Frequency
-| IAM ListAccountAliases | 1 | Once on startup
-| STS GetCallerIdentity | 1 | Once on startup
-| EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
-| CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per collection period
-|===
-
-[float]
-=== `s3_daily_storage` and `s3_request`
-|===
-| AWS API Name | AWS API Count | Frequency
-| IAM ListAccountAliases | 1 | Once on startup
-| STS GetCallerIdentity | 1 | Once on startup
-| EC2 DescribeRegions| 1 | Once on startup
-| CloudWatch ListMetrics | Total number of results / ListMetrics max page size | Per region per collection period
-| CloudWatch GetMetricData | Total number of results / GetMetricData max page size | Per region per collection period
-|===
-
 [id="aws-credentials-config"]
 include::{libbeat-xpack-dir}/docs/aws-credentials-config.asciidoc[]
 

metricbeat/mb/testing/testdata.go

@@ -344,7 +344,15 @@ func checkDocumented(data []common.MapStr, omitFields []string) error {
 }
 
 func documentedFieldCheck(foundKeys common.MapStr, knownKeys map[string]interface{}, omitFields []string) error {
-	for foundKey := range foundKeys {
+	// Sort all found keys to guarantee consistent validation messages
+	sortedFoundKeys := make([]string, 0, len(foundKeys))
+	for k := range foundKeys {
+		sortedFoundKeys = append(sortedFoundKeys, k)
+	}
+	sort.Strings(sortedFoundKeys)
+
+	for k := range sortedFoundKeys {
+		foundKey := sortedFoundKeys[k]
 		if _, ok := knownKeys[foundKey]; !ok {
 			for _, omitField := range omitFields {
 				if omitDocumentedField(foundKey, omitField) {

metricbeat/module/beat/beat.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/url"
+	"path"
+	"strings"
 
 	"github.com/elastic/beats/v7/libbeat/logp"
 	"github.com/elastic/beats/v7/metricbeat/helper"
@@ -35,9 +37,11 @@ func init() {
 	}
 }
 
+var metricSets = []string{"state", "stats"}
+
 // NewModule creates a new module
 func NewModule(base mb.BaseModule) (mb.Module, error) {
-	return elastic.NewModule(&base, []string{"state", "stats"}, logp.NewLogger(ModuleName))
+	return elastic.NewModule(&base, metricSets, logp.NewLogger(ModuleName))
 }
 
 // ModuleName is the name of this module.
@@ -76,7 +80,7 @@ type State struct {
 
 // GetInfo returns the data for the Beat's / endpoint.
 func GetInfo(m *MetricSet) (*Info, error) {
-	content, err := fetchPath(m.HTTP, "/", "")
+	content, err := fetchPath(m.HTTP, "/")
 	if err != nil {
 		return nil, err
 	}
@@ -92,7 +96,7 @@ func GetInfo(m *MetricSet) (*Info, error) {
 
 // GetState returns the data for the Beat's /state endpoint.
 func GetState(m *MetricSet) (*State, error) {
-	content, err := fetchPath(m.HTTP, "/state", "")
+	content, err := fetchPath(m.HTTP, "/state")
 	if err != nil {
 		return nil, err
 	}
@@ -106,16 +110,28 @@ func GetState(m *MetricSet) (*State, error) {
 	return info, nil
 }
 
-func fetchPath(httpHelper *helper.HTTP, path string, query string) ([]byte, error) {
+func fetchPath(httpHelper *helper.HTTP, path string) ([]byte, error) {
 	currentURI := httpHelper.GetURI()
 	defer httpHelper.SetURI(currentURI)
 
 	// Parses the uri to replace the path
-	u, _ := url.Parse(currentURI)
-	u.Path = path
-	u.RawQuery = query
+	u, err := url.Parse(currentURI)
+	if err != nil {
+		return nil, err
+	}
 
-	// Http helper includes the HostData with username and password
-	httpHelper.SetURI(u.String())
+	// HTTP helper includes the HostData with username and password
+	httpHelper.SetURI(fetchURI(u, path))
 	return httpHelper.FetchContent()
 }
+
+func fetchURI(u *url.URL, uriPath string) string {
+	for _, s := range metricSets {
+		if strings.HasSuffix(u.Path, s) {
+			u.Path = u.Path[:len(u.Path)-len(s)]
+			break
+		}
+	}
+	u.Path = path.Join(u.Path, uriPath)
+	return u.String()
+}