Merge remote-tracking branch 'upstream/master' into update_go_winio_lib

elastic · Oct 25, 2021 · c0ee156 · c0ee156
2 parents 911ebd7 + 444ae15
commit c0ee156
Show file tree

Hide file tree

Showing 337 changed files with 10,020 additions and 4,233 deletions.
diff --git a/.backportrc.json b/.backportrc.json
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -416,38 +416,14 @@ def runE2ETests(){
       };
     }
 
-    triggerE2ETests(suites)
+    runE2E(runTestsSuites: suites,
+           beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
+           gitHubCheckName: env.GITHUB_CHECK_E2E_TESTS_NAME,
+           gitHubCheckRepo: env.REPO,
+           gitHubCheckSha1: env.GIT_BASE_COMMIT)
   }
 }
 
-def triggerE2ETests(String suite) {
-  echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")
-
-  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
-  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
-  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
-
-  def parameters = [
-    booleanParam(name: 'forceSkipGitChecks', value: true),
-    booleanParam(name: 'forceSkipPresubmit', value: true),
-    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
-    string(name: 'BEAT_VERSION', value: beatVersion),
-    string(name: 'runTestsSuites', value: suite),
-    string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
-    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
-  ]
-
-  build(job: "${e2eTestsPipeline}",
-    parameters: parameters,
-    propagate: false,
-    wait: false
-  )
-
-  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME}"
-  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
-}
-
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],

diff --git a/.ci/schedule-daily.groovy b/.ci/schedule-daily.groovy
@@ -21,8 +21,8 @@ pipeline {
     stage('Nighly beats builds') {
       steps {
         runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.x')
-        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.<minor>')
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
+        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.15')
       }
     }
   }
@@ -35,9 +35,5 @@ pipeline {
 
 def runBuild(Map args = [:]) {
   def jobName = args.job
-  if (jobName.contains('7.<minor>')) {
-    def parts = stackVersions.release().split('\\.')
-    jobName = args.job.replaceAll('<minor>', parts[1])
-  }
   build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
 }
diff --git a/.ci/schedule-weekly.groovy b/.ci/schedule-weekly.groovy
@@ -18,12 +18,11 @@ pipeline {
     cron('H H(1-2) * * 0')
   }
   stages {
-    stage('Nighly beats builds') {
+    stage('Weekly beats builds') {
       steps {
         runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.x')
-        // <minor> is an alias to be replaced with the latest release branch
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.<minor>')
+        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.16')
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.15')
       }
     }
   }
@@ -36,9 +35,5 @@ pipeline {
 
 def runBuild(Map args = [:]) {
   def jobName = args.job
-  if (jobName.contains('7.<minor>')) {
-    def parts = stackVersions.release().split('\\.')
-    jobName = args.job.replaceAll('<minor>', parts[1])
-  }
   build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
 }
diff --git a/.mergify.yml b/.mergify.yml
@@ -26,9 +26,9 @@ pull_request_rules:
             git merge upstream/{{base}}
             git push upstream {{head}}
             ```
-  - name: automatic approval for mergify pull requests with bump updates
+  - name: automatic approval for automated pull requests with bump updates
     conditions:
-      - author=mergify[bot]
+      - author=apmmachine
       - check-success=beats-ci/pr-merge
       - label=automation
       - files~=^testing/environments/snapshot.*\.yml$
@@ -45,7 +45,7 @@ pull_request_rules:
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
       - or:
@@ -89,7 +89,7 @@ pull_request_rules:
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed
     conditions:
       - or:
@@ -124,42 +124,42 @@ pull_request_rules:
       label:
         remove:
           - backport-skip
-  - name: backport patches to 7.x branch
+  - name: backport patches to 7.14 branch
     conditions:
       - merged
-      - label=backport-v7.16.0
+      - label=backport-v7.14.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.x"
+          - "7.14"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
-  - name: backport patches to 7.14 branch
+  - name: backport patches to 7.15 branch
     conditions:
       - merged
-      - label=backport-v7.14.0
+      - label=backport-v7.15.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.14"
+          - "7.15"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
-  - name: backport patches to 7.15 branch
+  - name: backport patches to 7.16 branch
     conditions:
       - merged
-      - label=backport-v7.15.0
+      - label=backport-v7.16.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.15"
+          - "7.16"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+    -   id: check-merge-conflict
+
+-   repo: https://github.com/elastic/apm-pipeline-library.git
+    rev: current
+    hooks:
+    -   id: check-jenkins-pipelines
+        files: ^(.ci/(.*\.groovy|Jenkinsfile)|Jenkinsfile)$
+    -   id: check-jjbb
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -23,14 +23,17 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated/undocumented IncludeCreatorMetadata setting from kubernetes metadata config options {pull}28006[28006]
 - Remove deprecated fields from kubernetes module {pull}28046[28046]
 - Remove deprecated config option aws_partition. {pull}28120[28120]
+- Improve stats API {pull}27963[27963]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
 - Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
 
 *Auditbeat*
 
 - File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630]
 - Change event.kind=error to event.kind=event to comply with ECS. {issue}18870[18870] {pull}20685[20685]
-
+- File integrity dataset: Remove non-ECS `hash.*` fields. Hashes are under `file.hash.*`. {issue}19039[19039] {pull}28378[28378]
+- Auditd dataset: Removes the authentication_success and authentication_failure event.type values for user logins. {issue}19039[19039] {pull}28378[28378]
+- Fix handling of long file names on Windows. {issue}25334[25334] {pull}28517[28517]
 
 *Filebeat*
 
@@ -94,6 +97,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated fields in coredns module. {pull}28196[28196]
 - Remove old `httpjson` config implementation. {pull}28054[28054]
+- Added dataset `threatq` to the `threatintel` module to ingest indicators from ThreatQ {issue}27423[27423]
 
 *Heartbeat*
 
@@ -116,15 +120,20 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933]
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated config option default_region from aws module. {pull}28120[28120]
+- Remove network and diskio metrics from ec2 metricset. {pull}28316[28316]
+- Rename read/write_io.ops_per_sec to read/write.iops in rds metricset. {pull}28350[28350]
+- Remove linux-only metrics from diskio, memory {pull}28292[28292]
 - Remove deprecated config option perfmon.counters from windows/perfmon metricset. {pull}28282[28282]
 - Remove deprecated fields in Redis module. {issue}11835[11835] {pull}28246[28246]
+- Align fields to Beats naming conventions in GCP module. {issue}27231[27231] {pull}27974[27974]
 
 *Packetbeat*
 
 - Redis: fix incorrectly handle with two-words redis command. {issue}14872[14872] {pull}14873[14873]
 - `event.category` no longer contains the value `network_traffic` because this is not a valid ECS event category value. {pull}20556[20556]
 - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
 - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
+- Remove deprecated TLS fields in favor of tls.server.x509 and tls.client.x509 ECS fields. {pull}28487[28487]
 
 *Winlogbeat*
 
@@ -320,6 +329,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Relax time parsing and capture group and session type in Cisco ASA module {issue}24710[24710] {pull}28325[28325]
 - Correctly track bytes read when max_bytes is exceeded. {issue}28317[28317] {pull}28352[28352]
 - Fix initialization of http client in Cloudfoundry input. {issue}28271[28271] {pull}28277[28277]
+- Fix aws-s3 input by checking if GetObject API call response content type exists. {pull}28457[28457]
 
 *Heartbeat*
 
@@ -429,10 +439,12 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - `beat` module respects `basepath` config option. {pull}28162[28162]
 - Fix list_docker.go {pull}28374[28374]
+- Divide RDS metric cpu.total.pct by 100. {pull}28456[28456]
 
 *Packetbeat*
 
 - Handle truncated DNS records more gracefully. {issue}21495[21495] {pull}28297[28297]
+- Fix data stream name for network flows when running under Elastic Agent and Fleet. {pull}28408[28408]
 
 *Winlogbeat*
 
@@ -442,6 +454,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Protect against accessing undefined variables in Sysmon module. {issue}22219[22219] {pull}22236[22236]
 - Protect against accessing an undefined variable in Security module. {pull}22937[22937]
 - Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
+- Add ECS 1.9 new users fields {pull}26509[26509]
 
 *Functionbeat*
 
@@ -762,13 +775,17 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 - Release zoom module as GA. {pull}28106[28106]
 - Add support for secondary object attribute handling in ThreatIntel MISP module {pull}28124[28124]
+- Azure signinlogs - Add support for ManagedIdentitySignInLogs, NonInteractiveUserSignInLogs, and ServicePrincipalSignInLogs. {issue}23653[23653]
 - Add `base64Decode` and `base64DecodeNoPad` functions to `httpsjon` templates. {pull}28385[28385]
+- Add latency config option for aws-cloudwatch input. {pull}28509[28509]
+- Added proxy support to threatintel/malwarebazaar. {pull}28533[28533]
 
 *Heartbeat*
 
 - Add mime type detection for http responses. {pull}22976[22976]
 - Support JSON expressions / validation of JSON arrays. {pull}28073[28073]
 - Experimental 'run once' mode. {pull}25972[25972]
+- Add `keyword` multi-field mapping for `synthetics.step.name`. {pull}28452[28452]
 
 *Journalbeat*
 
@@ -896,6 +913,8 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Move openmetrics module to oss. {pull}26561[26561]
 - Add `gke` metricset collection to `gcp` module {pull}26824[26824]
 - Added a new beta `enterprisesearch` module for Elastic Enterprise Search {pull}27549[27549]
+- Preliminary AIX support {pull}27954[27954]
+- Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
 
 *Packetbeat*