Merge branch 'master' into feature/agent_actions_response

.ci/jobs/beats-schedule-daily.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: beats-schedule-daily
+    display-name: Jobs scheduled daily (weekdays)
+    description: Jobs scheduled daily (weekdays)
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-daily.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(2-3) * * 1-5'

.ci/jobs/beats-schedule-weekly.yml

@@ -24,4 +24,4 @@
             branches:
               - $branch_specifier
     triggers:
-      - timed: 'H H(1-4) * * 0'
+      - timed: 'H H(1-2) * * 0'

.ci/packaging.groovy

@@ -416,38 +416,14 @@ def runE2ETests(){
       };
     }
 
-    triggerE2ETests(suites)
+    runE2E(runTestsSuites: suites,
+           beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
+           gitHubCheckName: env.GITHUB_CHECK_E2E_TESTS_NAME,
+           gitHubCheckRepo: env.REPO,
+           gitHubCheckSha1: env.GIT_BASE_COMMIT)
   }
 }
 
-def triggerE2ETests(String suite) {
-  echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")
-
-  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
-  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
-  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
-
-  def parameters = [
-    booleanParam(name: 'forceSkipGitChecks', value: true),
-    booleanParam(name: 'forceSkipPresubmit', value: true),
-    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
-    string(name: 'BEAT_VERSION', value: beatVersion),
-    string(name: 'runTestsSuites', value: suite),
-    string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
-    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
-  ]
-
-  build(job: "${e2eTestsPipeline}",
-    parameters: parameters,
-    propagate: false,
-    wait: false
-  )
-
-  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME}"
-  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
-}
-
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],

.ci/schedule-daily.groovy

@@ -0,0 +1,43 @@
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    cron('H H(2-3) * * 1-5')
+  }
+  stages {
+    stage('Nighly beats builds') {
+      steps {
+        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.x')
+        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.<minor>')
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult(prComment: false)
+    }
+  }
+}
+
+def runBuild(Map args = [:]) {
+  def jobName = args.job
+  if (jobName.contains('7.<minor>')) {
+    def parts = stackVersions.release().split('\\.')
+    jobName = args.job.replaceAll('<minor>', parts[1])
+  }
+  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+}

.ci/schedule-weekly.groovy

@@ -15,13 +15,15 @@ pipeline {
     durabilityHint('PERFORMANCE_OPTIMIZED')
   }
   triggers {
-    cron('H H(1-4) * * 0')
+    cron('H H(1-2) * * 0')
   }
   stages {
     stage('Nighly beats builds') {
       steps {
-        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
-        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.x')
+        // <minor> is an alias to be replaced with the latest release branch
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.<minor>')
       }
     }
   }
@@ -31,3 +33,12 @@ pipeline {
     }
   }
 }
+
+def runBuild(Map args = [:]) {
+  def jobName = args.job
+  if (jobName.contains('7.<minor>')) {
+    def parts = stackVersions.release().split('\\.')
+    jobName = args.job.replaceAll('<minor>', parts[1])
+  }
+  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+}

.mergify.yml

@@ -26,9 +26,9 @@ pull_request_rules:
             git merge upstream/{{base}}
             git push upstream {{head}}
             ```
-  - name: automatic approval for mergify pull requests with bump updates
+  - name: automatic approval for automated pull requests with bump updates
     conditions:
-      - author=mergify[bot]
+      - author=apmmachine
       - check-success=beats-ci/pr-merge
       - label=automation
       - files~=^testing/environments/snapshot.*\.yml$
@@ -41,10 +41,11 @@ pull_request_rules:
       - check-success=beats-ci/pr-merge
       - label=automation
       - files~=^testing/environments/snapshot.*\.yml$
+      - "#approved-reviews-by>=1"
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
       - or:
@@ -84,10 +85,11 @@ pull_request_rules:
       - label=automation
       - files~=^\.mergify\.yml$
       - head~=^add-backport-next.*
+      - "#approved-reviews-by>=1"
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed
     conditions:
       - or:
@@ -122,42 +124,42 @@ pull_request_rules:
       label:
         remove:
           - backport-skip
-  - name: backport patches to 7.x branch
+  - name: backport patches to 7.14 branch
     conditions:
       - merged
-      - label=backport-v7.16.0
+      - label=backport-v7.14.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.x"
+          - "7.14"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
-  - name: backport patches to 7.14 branch
+  - name: backport patches to 7.15 branch
     conditions:
       - merged
-      - label=backport-v7.14.0
+      - label=backport-v7.15.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.14"
+          - "7.15"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
-  - name: backport patches to 7.15 branch
+  - name: backport patches to 7.16 branch
     conditions:
       - merged
-      - label=backport-v7.15.0
+      - label=backport-v7.16.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.15"
+          - "7.16"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+    -   id: check-merge-conflict
+
+-   repo: https://github.com/elastic/apm-pipeline-library.git
+    rev: current
+    hooks:
+    -   id: check-jenkins-pipelines
+        files: ^(.ci/(.*\.groovy|Jenkinsfile)|Jenkinsfile)$
+    -   id: check-jjbb

CHANGELOG-developer.next.asciidoc

@@ -55,6 +55,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
+- Removed the `common.Float` type. {issue}28279[28279] {pull}28280[28280] {pull}28376[28376]
 
 ==== Bugfixes
 
@@ -123,6 +124,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]
 - Introduce `libbeat/beat.Beat.OutputConfigReloader` {pull}28048[28048]
 - Update Go version to 1.17.1. {pull}27543[27543]
+- Whitelist `GCP_*` environment variables in dev tools {pull}28364[28364]
 
 ==== Deprecated
 

CHANGELOG.asciidoc

@@ -12,6 +12,21 @@ Changes will be described in a later alpha / beta.
 === Beats version 8.0.0-alpha1
 
 Changes will be described in a later alpha / beta.
+[[release-notes-7.15.1]]
+=== Beats version 7.15.1
+https://github.com/elastic/beats/compare/v7.15.0...v7.15.1[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Update Sophos xg module pipeline to deal with missing `date` and `time` fields. {pull}27834[27834]
+- sophos/xg fileset: Add missing pipeline for System Health logs. {pull}27827[27827] {issue}27826[27826]
+
+*Metricbeat*
+
+- Add support for kube-state-metrics v2.0.0. {pull}27552[27552]
+
 [[release-notes-7.15.0]]
 === Beats version 7.15.0
 https://github.com/elastic/beats/compare/v7.14.2...v7.15.0[View commits]
@@ -54,6 +69,7 @@ https://github.com/elastic/beats/compare/v7.14.2...v7.15.0[View commits]
 *Auditbeat*
 
 - File Integrity Module: Honor `include_files` when doing initial scan. {issue}27273[27273] {pull}27722[27722]
+- Fix handling of root and relative paths {issue}24430[24430] {pull}28354[28354]
 
 *Filebeat*