Merge remote-tracking branch 'upstream/master' into feature/use-debian-9

* upstream/master: (577 commits)
  Remove Journalbeat (elastic#29131)
  Add note that there is no warranty or support for generator code (elastic#28797)
  packetbeat: preparation for npcap addition (elastic#29017)
  Use the generic helper for opening file to read in filestream (elastic#29180)
  Workflow for macos (elastic#29174)
  Fix `decode_json_fields` processor to always add error key (elastic#29107)
  Workflow for macos (elastic#29156)
  Fix agent download timeout values in yaml files (elastic#29039)
  Added workflow file for builds with macos (elastic#29148)
  CI: enable AWS cloud testing on a PR basis if changes in certain files (elastic#29047)
  Remove links to Journalbeat (elastic#29134)
  Fix rds metadata in cloudwatch metricset (elastic#29106)
  [mergify]: notify conflicts in PRs that are still open (elastic#29122)
  Use NamedWatcher in Agent's k8s provider (elastic#29095)
  override host on statsd metricset (elastic#29103)
  Skip config check in autodiscover for duplicated configurations (elastic#29048)
  Change "filebeat.config.modules.enabled" to "true" (elastic#28769)
  Remove deprecated spool queue from Beats (elastic#28869)
  Add `beat` field back to beat.stats (elastic#29094)
  Revert "Move labels and annotations under kubernetes.namespace. (elastic#27917)" (elastic#29069)
  ...

.ci/apm-beats-update.groovy

@@ -126,7 +126,9 @@ def beatsUpdate() {
         git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"
 
         go mod edit -replace github.com/elastic/beats/v7=\${GOPATH}/src/github.com/elastic/beats-local
+        go mod tidy
         echo '{"name": "${GOPATH}/src/github.com/elastic/beats-local", "licenceType": "Elastic"}' >> \${GOPATH}/src/github.com/elastic/beats-local/dev-tools/notice/overrides.json
+
         make update
         git commit -a -m beats-update
 

.ci/jobs/beats-schedule-daily.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: beats-schedule-daily
+    display-name: Jobs scheduled daily (weekdays)
+    description: Jobs scheduled daily (weekdays)
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-daily.groovy
+      scm:
+        - git:
+            url: [email protected]:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(2-3) * * 1-5'

.ci/jobs/beats-schedule-weekly.yml

@@ -24,4 +24,4 @@
             branches:
               - $branch_specifier
     triggers:
-      - timed: 'H H(1-4) * * 0'
+      - timed: 'H H(1-2) * * 0'

.ci/jobs/beats.yml

@@ -17,7 +17,7 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|6\.[89]|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+        head-filter-regex: '(master|6\.[89]|7\.16|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
         repo: 'beats'

.ci/packaging.groovy

@@ -40,8 +40,6 @@ pipeline {
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
-    booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
-    booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
   }
   stages {
     stage('Filter build') {
@@ -103,7 +101,6 @@ pipeline {
                   'auditbeat',
                   'filebeat',
                   'heartbeat',
-                  'journalbeat',
                   'metricbeat',
                   'packetbeat',
                   'winlogbeat',
@@ -113,7 +110,6 @@ pipeline {
                   'x-pack/filebeat',
                   'x-pack/functionbeat',
                    'x-pack/heartbeat',
-                  // 'x-pack/journalbeat',
                   'x-pack/metricbeat',
                   'x-pack/osquerybeat',
                   'x-pack/packetbeat',
@@ -125,12 +121,6 @@ pipeline {
               stage('Package Linux'){
                 agent { label 'ubuntu-18 && immutable' }
                 options { skipDefaultCheckout() }
-                when {
-                  beforeAgent true
-                  expression {
-                    return params.linux
-                  }
-                }
                 environment {
                   HOME = "${env.WORKSPACE}"
                   PLATFORMS = [
@@ -207,7 +197,6 @@ pipeline {
                   'auditbeat',
                   'filebeat',
                   'heartbeat',
-                  'journalbeat',
                   'metricbeat',
                   'packetbeat',
                   'x-pack/auditbeat',
@@ -224,12 +213,6 @@ pipeline {
               stage('Package Docker images for linux/arm64'){
                 agent { label 'arm' }
                 options { skipDefaultCheckout() }
-                when {
-                  beforeAgent true
-                  expression {
-                    return params.arm
-                  }
-                }
                 environment {
                   HOME = "${env.WORKSPACE}"
                   PACKAGES = "docker"
@@ -291,8 +274,6 @@ def pushCIDockerImages(Map args = [:]) {
       tagAndPush(beatName: 'filebeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
       tagAndPush(beatName: 'heartbeat', arch: arch)
-    } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
-      tagAndPush(beatName: 'journalbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
       tagAndPush(beatName: 'metricbeat', arch: arch)
     } else if (env?.BEATS_FOLDER?.endsWith('osquerybeat')) {
@@ -335,22 +316,31 @@ def tagAndPush(Map args = [:]) {
   }
   // supported image flavours
   def variants = ["", "-oss", "-ubi8"]
+
+  if(beatName == 'elastic-agent'){
+      variants.add("-complete")
+      variants.add("-cloud")
+  }
+
   variants.each { variant ->
+    // cloud docker images are stored in the private docker namespace.
+    def sourceNamespace = variant.equals('-cloud') ? 'beats-ci' : 'beats'
     tags.each { tag ->
       // TODO:
       // For backward compatibility let's ensure we tag only for amd64, then E2E can benefit from until
       // they support the versioning with the architecture
       if ("${arch}" == "amd64") {
-        doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}")
+        doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}", sourceNamespace: sourceNamespace)
       }
-      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}")
+      doTagAndPush(beatName: beatName, variant: variant, sourceTag: libbetaVer, targetTag: "${tag}-${arch}", sourceNamespace: sourceNamespace)
     }
   }
 }
 
 /**
 * @param beatName name of the Beat
 * @param variant name of the variant used to build the docker image name
+* @param sourceNamespace namespace to be used as source for the docker tag command
 * @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
 * @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
 */
@@ -359,7 +349,8 @@ def doTagAndPush(Map args = [:]) {
   def variant = args.variant
   def sourceTag = args.sourceTag
   def targetTag = args.targetTag
-  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def sourceNamespace = args.sourceNamespace
+  def sourceName = "${DOCKER_REGISTRY}/${sourceNamespace}/${beatName}${variant}:${sourceTag}"
   def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
   def iterations = 0
   retryWithSleep(retries: 3, seconds: 5, backoff: true) {
@@ -420,39 +411,14 @@ def runE2ETests(){
       };
     }
 
-    triggerE2ETests(suites)
+    runE2E(runTestsSuites: suites,
+           beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
+           gitHubCheckName: env.GITHUB_CHECK_E2E_TESTS_NAME,
+           gitHubCheckRepo: env.REPO,
+           gitHubCheckSha1: env.GIT_BASE_COMMIT)
   }
 }
 
-def triggerE2ETests(String suite) {
-  echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")
-
-  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
-  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
-  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
-
-  def parameters = [
-    booleanParam(name: 'forceSkipGitChecks', value: true),
-    booleanParam(name: 'forceSkipPresubmit', value: true),
-    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
-    string(name: 'BEAT_VERSION', value: beatVersion),
-    booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
-    string(name: 'runTestsSuites', value: suite),
-    string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
-    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
-  ]
-
-  build(job: "${e2eTestsPipeline}",
-    parameters: parameters,
-    propagate: false,
-    wait: false
-  )
-
-  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME}"
-  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
-}
-
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],

.ci/schedule-daily.groovy

@@ -0,0 +1,42 @@
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    cron('H H(2-3) * * 1-5')
+  }
+  stages {
+    stage('Nighly beats builds') {
+      steps {
+        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+        // This should be `current_8` bump.getCurrentMinorReleaseFor8
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/8.0')
+        // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
+        // `next_minor_7`  bump.getNextMinorReleaseFor7
+        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.16')
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult(prComment: false)
+    }
+  }
+}
+
+def runBuild(Map args = [:]) {
+  def jobName = args.job
+  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
+}

.ci/schedule-weekly.groovy

@@ -15,13 +15,17 @@ pipeline {
     durabilityHint('PERFORMANCE_OPTIMIZED')
   }
   triggers {
-    cron('H H(1-4) * * 0')
+    cron('H H(1-2) * * 0')
   }
   stages {
-    stage('Nighly beats builds') {
+    stage('Weekly beats builds') {
       steps {
-        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
-        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+        runBuild(quietPeriod: 0, job: 'Beats/beats/master')
+        // This should be `current_8` bump.getCurrentMinorReleaseFor8
+        runBuild(quietPeriod: 1000, job: 'Beats/beats/8.0')
+        // This should be `current_7` bump.getCurrentMinorReleaseFor7 or
+        // `next_minor_7`  bump.getNextMinorReleaseFor7
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
       }
     }
   }
@@ -31,3 +35,8 @@ pipeline {
     }
   }
 }
+
+def runBuild(Map args = [:]) {
+  def jobName = args.job
+  build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+}

.ci/scripts/install-go.bat

@@ -0,0 +1,57 @@
+set GOPATH=%WORKSPACE%
+set MAGEFILE_CACHE=%WORKSPACE%\.magefile
+
+set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH%
+
+curl --version >nul 2>&1 && (
+    echo found curl
+) || (
+    choco install curl -y --no-progress --skipdownloadcache
+)
+
+mkdir %WORKSPACE%\bin
+
+IF EXIST "%PROGRAMFILES(X86)%" (
+    REM Force the gvm installation.
+    SET GVM_BIN=gvm.exe
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-amd64.exe
+    IF ERRORLEVEL 1 (
+        REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
+        exit /b 1
+    )
+) ELSE (
+    REM Windows 7 workers got a broken gvm installation.
+    curl -L -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.3.0/gvm-windows-386.exe
+    IF ERRORLEVEL 1 (
+        REM gvm installation has failed.
+        del bin\gvm.exe /s /f /q
+        exit /b 1
+    )
+)
+
+SET GVM_BIN=gvm.exe
+WHERE /q %GVM_BIN%
+%GVM_BIN% version
+
+REM Install the given go version
+%GVM_BIN% --debug install %GO_VERSION%
+
+REM Configure the given go version
+FOR /f "tokens=*" %%i IN ('"%GVM_BIN%" use %GO_VERSION% --format=batch') DO %%i
+
+go env
+IF ERRORLEVEL 1 (
+    REM go is not configured correctly.
+    rmdir %WORKSPACE%\.gvm /s /q
+    exit /b 1
+)
+
+where mage
+mage -version
+IF ERRORLEVEL 1 (
+    go get github.com/magefile/mage
+    IF ERRORLEVEL 1 (
+        exit /b 1
+    )
+)