Merge branch 'master' into remove_fields_ec2

.ci/jobs/beats-schedule-daily.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: beats-schedule-daily
+    display-name: Jobs scheduled daily (weekdays)
+    description: Jobs scheduled daily (weekdays)
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-daily.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(2-3) * * 1-5'

.ci/jobs/beats-schedule-weekly.yml

@@ -24,4 +24,4 @@
             branches:
               - $branch_specifier
     triggers:
-      - timed: 'H H(1-4) * * 0'
+      - timed: 'H H(1-2) * * 0'

.ci/schedule-daily.groovy

@@ -0,0 +1,43 @@
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    cron('H H(2-3) * * 1-5')
+  }
+  stages {
+    stage('Nighly beats builds') {
+      steps {
+        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.x')
+        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.<minor>')
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult(prComment: false)
+    }
+  }
+}
+
+def runBuild(Map args = [:]) {
+  def jobName = args.job
+  if (jobName.contains('7.<minor>')) {
+    def parts = stackVersions.release().split('\\.')
+    jobName = args.job.replaceAll('<minor>', parts[1])
+  }
+  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+}

.ci/schedule-weekly.groovy

@@ -15,13 +15,15 @@ pipeline {
     durabilityHint('PERFORMANCE_OPTIMIZED')
   }
   triggers {
-    cron('H H(1-4) * * 0')
+    cron('H H(1-2) * * 0')
   }
   stages {
     stage('Nighly beats builds') {
       steps {
-        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
-        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.x')
+        // <minor> is an alias to be replaced with the latest release branch
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.<minor>')
       }
     }
   }
@@ -31,3 +33,12 @@ pipeline {
     }
   }
 }
+
+def runBuild(Map args = [:]) {
+  def jobName = args.job
+  if (jobName.contains('7.<minor>')) {
+    def parts = stackVersions.release().split('\\.')
+    jobName = args.job.replaceAll('<minor>', parts[1])
+  }
+  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+}

.go-version

@@ -1 +1 @@
-1.16.6
+1.17.1

.mergify.yml

@@ -26,9 +26,9 @@ pull_request_rules:
             git merge upstream/{{base}}
             git push upstream {{head}}
             ```
-  - name: automatic approval for mergify pull requests with bump updates
+  - name: automatic approval for automated pull requests with bump updates
     conditions:
-      - author=mergify[bot]
+      - author=apmmachine
       - check-success=beats-ci/pr-merge
       - label=automation
       - files~=^testing/environments/snapshot.*\.yml$
@@ -41,10 +41,11 @@ pull_request_rules:
       - check-success=beats-ci/pr-merge
       - label=automation
       - files~=^testing/environments/snapshot.*\.yml$
+      - "#approved-reviews-by>=1"
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
       - or:
@@ -84,10 +85,11 @@ pull_request_rules:
       - label=automation
       - files~=^\.mergify\.yml$
       - head~=^add-backport-next.*
+      - "#approved-reviews-by>=1"
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed
     conditions:
       - or:

CHANGELOG-developer.next.asciidoc

@@ -55,6 +55,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
+- Removed the `common.Float` type. {issue}28279[28279] {pull}28280[28280] {pull}28376[28376]
 
 ==== Bugfixes
 
@@ -122,6 +123,8 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update to go-concert 0.2.0 {pull}27162[27162]
 - Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]
 - Introduce `libbeat/beat.Beat.OutputConfigReloader` {pull}28048[28048]
+- Update Go version to 1.17.1. {pull}27543[27543]
+- Whitelist `GCP_*` environment variables in dev tools {pull}28364[28364]
 
 ==== Deprecated
 

CHANGELOG.asciidoc

@@ -12,6 +12,21 @@ Changes will be described in a later alpha / beta.
 === Beats version 8.0.0-alpha1
 
 Changes will be described in a later alpha / beta.
+[[release-notes-7.15.1]]
+=== Beats version 7.15.1
+https://github.com/elastic/beats/compare/v7.15.0...v7.15.1[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Update Sophos xg module pipeline to deal with missing `date` and `time` fields. {pull}27834[27834]
+- sophos/xg fileset: Add missing pipeline for System Health logs. {pull}27827[27827] {issue}27826[27826]
+
+*Metricbeat*
+
+- Add support for kube-state-metrics v2.0.0. {pull}27552[27552]
+
 [[release-notes-7.15.0]]
 === Beats version 7.15.0
 https://github.com/elastic/beats/compare/v7.14.2...v7.15.0[View commits]
@@ -54,6 +69,7 @@ https://github.com/elastic/beats/compare/v7.14.2...v7.15.0[View commits]
 *Auditbeat*
 
 - File Integrity Module: Honor `include_files` when doing initial scan. {issue}27273[27273] {pull}27722[27722]
+- Fix handling of root and relative paths {issue}24430[24430] {pull}28354[28354]
 
 *Filebeat*
 

CHANGELOG.next.asciidoc

@@ -23,12 +23,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated/undocumented IncludeCreatorMetadata setting from kubernetes metadata config options {pull}28006[28006]
 - Remove deprecated fields from kubernetes module {pull}28046[28046]
 - Remove deprecated config option aws_partition. {pull}28120[28120]
+- Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
+- Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
 
 *Auditbeat*
 
 - File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630]
 - Change event.kind=error to event.kind=event to comply with ECS. {issue}18870[18870] {pull}20685[20685]
-
+- File integrity dataset: Remove non-ECS `hash.*` fields. Hashes are under `file.hash.*`. {issue}19039[19039] {pull}28378[28378]
+- Auditd dataset: Removes the authentication_success and authentication_failure event.type values for user logins. {issue}19039[19039] {pull}28378[28378]
 
 *Filebeat*
 
@@ -92,7 +95,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated fields in coredns module. {pull}28196[28196]
 - Remove old `httpjson` config implementation. {pull}28054[28054]
-- Fix initialization of http client in Cloudfoundry input. {issue}28271[28271] {pull}28277[28277]
+- Added dataset `threatq` to the `threatintel` module to ingest indicators from ThreatQ {issue}27423[27423]
 
 *Heartbeat*
 
@@ -110,20 +113,25 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix ECS compliance of user.id field in system/users  metricset {pull}19019[19019]
 - Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457]
 - Add Linux pressure metricset {pull}27355[27355]
-- Add support for kube-state-metrics v2.0.0 {pull}27552[27552]
 - Add User-Agent header to HTTP requests. {issue}18160[18160] {pull}27509[27509]
 - Errors should be thrown as errors. Metricsets inside Metricbeat will now throw errors as the `error` log level. {pull}27804[27804]
 - Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933]
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated config option default_region from aws module. {pull}28120[28120]
 - Remove network and diskio metrics from ec2 metricset. {pull}28316[28316]
+- Rename read/write_io.ops_per_sec to read/write.iops in rds metricset. {pull}28350[28350]
+- Remove linux-only metrics from diskio, memory {pull}28292[28292]
+- Remove deprecated config option perfmon.counters from windows/perfmon metricset. {pull}28282[28282]
+- Remove deprecated fields in Redis module. {issue}11835[11835] {pull}28246[28246]
+- Align fields to Beats naming conventions in GCP module. {issue}27231[27231] {pull}27974[27974]
 
 *Packetbeat*
 
 - Redis: fix incorrectly handle with two-words redis command. {issue}14872[14872] {pull}14873[14873]
 - `event.category` no longer contains the value `network_traffic` because this is not a valid ECS event category value. {pull}20556[20556]
 - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
 - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
+- Remove deprecated TLS fields in favor of tls.server.x509 and tls.client.x509 ECS fields. {pull}28487[28487]
 
 *Winlogbeat*
 
@@ -140,6 +148,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 
+- Support for Google Cloud Functions have been removed, as it has been in Beta for a long time and been broken
+for a few releases. Please use other tools provided by Elastic to fetch data from GCP (e.g. Filebeat).
+
 
 ==== Bugfixes
 
@@ -212,6 +223,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
   as gauges (rather than counters). {pull}22877[22877]
 - Beats dashboards use custom index when `setup.dashboards.index` is set. {issue}21232[21232] {pull}27901[27901]
 - Fix handling of float data types within processors. {issue}28279[28279] {pull}28280[28280]
+- Allow `clone3` syscall in seccomp filters. {pull}28117[28117]
+- Remove unnecessary escaping step in dashboard loading, so they can be displayed in Kibana. {pull}28395[28395]
 
 *Auditbeat*
 
@@ -307,15 +320,22 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377]
 - Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
 - Update indentation for azure filebeat configuration. {pull}26604[26604]
-- Update Sophos xg module pipeline to deal with missing `date` and `time` fields. {pull}27834[27834]
-- sophos/xg fileset: Add missing pipeline for System Health logs. {pull}27827[27827] {issue}27826[27826]
+- Add support for passing a prefix on S3 bucket list mode for AWS-S3 input {pull}28252[28252] {issue}27965[27965]
 - Resolve issue with @timestamp for defender_atp. {pull}28272[28272]
 - Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
+- Add support for username in cisco asa security negotiation logs {pull}26975[26975]
+- Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
+- Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352]
+- Fix initialization of http client in Cloudfoundry input. {issue}28271[28271] {pull}28277[28277]
+- Fix aws-s3 input by checking if GetObject API call response content type exists. {pull}28457[28457]
 
 *Heartbeat*
 
 - Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 - Fixed TCP TLS checks to properly validate hostnames, this broke in 7.x and only worked for IP SANs. {pull}17549[17549]
+- Fix broken seccomp filtering and improve security via `setcap` and `setuid` when running as root on linux in containers. {pull}27878[27878]
+- Log browser `zip_url` download failures as `warn` instead of as `info`. {pull}28440[28440]
+- Properly locate base stream in fleet configs. {pull}28455[28455]
 
 *Journalbeat*
 
@@ -416,9 +436,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove io.time from windows {pull}22237[22237]
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - `beat` module respects `basepath` config option. {pull}28162[28162]
+- Fix list_docker.go {pull}28374[28374]
+- Divide RDS metric cpu.total.pct by 100. {pull}28456[28456]
 
 *Packetbeat*
 
+- Handle truncated DNS records more gracefully. {issue}21495[21495] {pull}28297[28297]
+- Fix data stream name for network flows when running under Elastic Agent and Fleet. {pull}28408[28408]
 
 *Winlogbeat*
 
@@ -428,6 +452,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Protect against accessing undefined variables in Sysmon module. {issue}22219[22219] {pull}22236[22236]
 - Protect against accessing an undefined variable in Security module. {pull}22937[22937]
 - Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
+- Add ECS 1.9 new users fields {pull}26509[26509]
 
 *Functionbeat*
 
@@ -519,6 +544,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update ECS to 1.12.0. {pull}27770[27770]
 - Fields mapped as `match_only_text` will automatically fallback to a `text` mapping when using Elasticsearch versions that do not support `match_only_text`. {pull}27770[27770]
 - Update cloud.google.com/go library. {pull}28229[28229]
+- Add additional metadata to the root HTTP endpoint. {pull}28265[28265]
 - Upgrade k8s.io/client-go library. {pull}28228[28228]
 
 *Auditbeat*
@@ -747,11 +773,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 - Release zoom module as GA. {pull}28106[28106]
 - Add support for secondary object attribute handling in ThreatIntel MISP module {pull}28124[28124]
+- Add `base64Decode` and `base64DecodeNoPad` functions to `httpsjon` templates. {pull}28385[28385]
+- Add latency config option for aws-cloudwatch input. {pull}28509[28509]
 
 *Heartbeat*
 
 - Add mime type detection for http responses. {pull}22976[22976]
 - Support JSON expressions / validation of JSON arrays. {pull}28073[28073]
+- Experimental 'run once' mode. {pull}25972[25972]
+- Add `keyword` multi-field mapping for `synthetics.step.name`. {pull}28452[28452]
 
 *Journalbeat*
 
@@ -879,6 +909,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move openmetrics module to oss. {pull}26561[26561]
 - Add `gke` metricset collection to `gcp` module {pull}26824[26824]
 - Added a new beta `enterprisesearch` module for Elastic Enterprise Search {pull}27549[27549]
+- Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
 
 *Packetbeat*