From a38cd76d8bc4eca860b20dd7b9efb517072ddbfd Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 15:41:31 +0000
Subject: [PATCH 01/49] [CI] Enable packaging with Pipeline 2.0

---
 Jenkinsfile                | 28 ++++++++++++++++++++++++++++
 metricbeat/Jenkinsfile.yml |  2 ++
 2 files changed, 30 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 072f2c3176a2..1eeec67da642 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -259,6 +259,31 @@ def k8sTest(Map args = [:]) {
   }
 }
 
+/**
+* This method runs the packaging
+*/
+def packagingLinux(Map args = [:]) {
+  def PLATFORMS = [ '+all',
+                'linux/amd64',
+                'linux/386',
+                'linux/arm64',
+                'linux/armv7',
+                // The platforms above are disabled temporarly as crossbuild images are
+                // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
+                //'linux/ppc64le',
+                //'linux/mips64',
+                //'linux/s390x',
+                'windows/amd64',
+                'windows/386',
+                (params.macos ? '' : 'darwin/amd64'),
+              ].join(' ')
+  withEnv([
+    "PLATFORMS=${PLATFORMS}"
+  ]) {
+    target(args)
+  }
+}
+
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false
@@ -714,6 +739,9 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
     if(args?.content?.containsKey('mage')) {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
     }
+    if(args?.content?.containsKey('packaging-linux')) {
+      steps.packagingLinux(context: args.context, command: args.content.mage, directory: args.project, label: args.label, isMage: true, id: args.id)
+    }
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
     }
diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
index 2bdb1218a5d2..57ab1986c71f 100644
--- a/metricbeat/Jenkinsfile.yml
+++ b/metricbeat/Jenkinsfile.yml
@@ -75,3 +75,5 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"

From 6e423abe0f97882dabc15577d58ffd5a4ad7bea2 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 16:18:39 +0000
Subject: [PATCH 02/49] Support e2e approach

---
 Jenkinsfile                | 37 ++++++++++++++++++++++++++++++++++++-
 metricbeat/Jenkinsfile.yml |  1 +
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1eeec67da642..25e973627ec2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -284,6 +284,36 @@ def packagingLinux(Map args = [:]) {
   }
 }
 
+def e2e(Map args = [:]) {
+  def suite = args.get('suite', '')
+  def tags = args.get('tags', '')
+  def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
+  dir('.e2e') {
+    gitCheckout(
+      branch: "master",  // TBC with the target branch if running on a PR basis.
+      repo: "https://github.com/elastic/e2e-testing.git",
+      credentialsId: "${JOB_GIT_CREDENTIALS}"
+    )
+    try {
+      if(isInstalled(tool: 'docker', flag: '--version')) {
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+      }
+      retry(3){
+        sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
+      }
+      filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
+        // TBC with the suite to be used
+        sh script: """SUITE=metricbeat DEVELOPER_MODE=false TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE make -C e2e functional-test""", label: "Run functional tests for ${suite}:${tags}"
+      }
+    } catch(e) {
+      error(e.toString())
+    } finally {
+      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
+        archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+    }
+  }
+}
+
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false
@@ -295,6 +325,7 @@ def target(Map args = [:]) {
   def directory = args.get('directory', '')
   def withModule = args.get('withModule', false)
   def isMage = args.get('isMage', false)
+  def isE2E = args.get('isE2E', false)
   withNode(args.label) {
     withGithubNotify(context: "${context}") {
       withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
@@ -304,6 +335,9 @@ def target(Map args = [:]) {
         dir(isMage ? directory : '') {
           cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
         }
+        if(isE2E) {
+          e2e(suite: directory)
+        }
       }
     }
   }
@@ -733,6 +767,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   }
   public run(Map args = [:]){
     def withModule = args.content.get('withModule', false)
+    def isE2E = args.content.get('isE2E', false)
     if(args?.content?.containsKey('make')) {
       steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
     }
@@ -740,7 +775,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
     }
     if(args?.content?.containsKey('packaging-linux')) {
-      steps.packagingLinux(context: args.context, command: args.content.mage, directory: args.project, label: args.label, isMage: true, id: args.id)
+      steps.packagingLinux(context: args.context, command: args.content.mage, directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
     }
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
index 57ab1986c71f..ed1edad19720 100644
--- a/metricbeat/Jenkinsfile.yml
+++ b/metricbeat/Jenkinsfile.yml
@@ -77,3 +77,4 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
+        isE2E: true

From f5bdbb698766a46ef9a5b7b71f106085aeda2f98 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 16:25:26 +0000
Subject: [PATCH 03/49] Support packaging with pipeline 2.0

---
 Jenkinsfile.yml                      |  4 ++--
 auditbeat/Jenkinsfile.yml            |  3 +++
 filebeat/Jenkinsfile.yml             |  3 +++
 heartbeat/Jenkinsfile.yml            |  3 +++
 journalbeat/Jenkinsfile.yml          |  3 +++
 packetbeat/Jenkinsfile.yml           |  3 +++
 winlogbeat/Jenkinsfile.yml           |  3 +++
 x-pack/auditbeat/Jenkinsfile.yml     |  3 +++
 x-pack/dockerlogbeat/Jenkinsfile.yml |  3 +++
 x-pack/elastic-agent/Jenkinsfile.yml |  3 +++
 x-pack/filebeat/Jenkinsfile.yml      |  3 +++
 x-pack/functionbeat/Jenkinsfile.yml  |  3 +++
 x-pack/heartbeat/Jenkinsfile.yml     | 18 ++++++++++++++++++
 x-pack/metricbeat/Jenkinsfile.yml    |  3 +++
 x-pack/packetbeat/Jenkinsfile.yml    |  3 +++
 x-pack/winlogbeat/Jenkinsfile.yml    |  5 +++++
 16 files changed, 64 insertions(+), 2 deletions(-)
 create mode 100644 x-pack/heartbeat/Jenkinsfile.yml

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
index 22fd31ae4b1e..284000da86fe 100644
--- a/Jenkinsfile.yml
+++ b/Jenkinsfile.yml
@@ -14,13 +14,13 @@ projects:
     - "x-pack/elastic-agent"
     - "x-pack/filebeat"
     - "x-pack/functionbeat"
+    - "x-pack/heartbeat"
     - "x-pack/libbeat"
     - "x-pack/metricbeat"
     - "x-pack/packetbeat"
     - "x-pack/winlogbeat"
     - "dev-tools"
-    ##- "x-pack/heartbeat"    It's not yet in the 1.0 pipeline.
-    ##- "x-pack/journalbeat"  It's not yet in the 1.0 pipeline.
+    ##- "x-pack/journalbeat"  It is not yet in the 1.0 pipeline.
 
 ## Changeset macros that are defined here and used in each specific 2.0 pipeline.
 changeset:
diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
index 168ac26edd50..52c3bf9f02ec 100644
--- a/auditbeat/Jenkinsfile.yml
+++ b/auditbeat/Jenkinsfile.yml
@@ -83,3 +83,6 @@ stages:
     #            - "windows-7"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
index ea1215b65140..e9e4bf55013c 100644
--- a/filebeat/Jenkinsfile.yml
+++ b/filebeat/Jenkinsfile.yml
@@ -81,3 +81,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
index de73acd5be79..4693c1ae3985 100644
--- a/heartbeat/Jenkinsfile.yml
+++ b/heartbeat/Jenkinsfile.yml
@@ -80,3 +80,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml
index f120d5f936d7..37b4ab233089 100644
--- a/journalbeat/Jenkinsfile.yml
+++ b/journalbeat/Jenkinsfile.yml
@@ -33,3 +33,6 @@ stages:
             tags: true         ## for all the tags
     unitTest:
         mage: "mage build unitTest"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
index 0225058ab1c7..f00a129d4d7e 100644
--- a/packetbeat/Jenkinsfile.yml
+++ b/packetbeat/Jenkinsfile.yml
@@ -81,3 +81,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml
index 44ee6c06513c..047cda73007d 100644
--- a/winlogbeat/Jenkinsfile.yml
+++ b/winlogbeat/Jenkinsfile.yml
@@ -51,3 +51,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
index deb1fdddd3a4..c594fb5990c8 100644
--- a/x-pack/auditbeat/Jenkinsfile.yml
+++ b/x-pack/auditbeat/Jenkinsfile.yml
@@ -81,3 +81,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml
index bc430fbf89f4..12db0b0c3efe 100644
--- a/x-pack/dockerlogbeat/Jenkinsfile.yml
+++ b/x-pack/dockerlogbeat/Jenkinsfile.yml
@@ -21,3 +21,6 @@ stages:
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml
index 0e6ecef9e4cd..b6daada774b3 100644
--- a/x-pack/elastic-agent/Jenkinsfile.yml
+++ b/x-pack/elastic-agent/Jenkinsfile.yml
@@ -85,3 +85,6 @@ stages:
     #            - "windows-7-32"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
index 0581f2cd8600..516100eb038f 100644
--- a/x-pack/filebeat/Jenkinsfile.yml
+++ b/x-pack/filebeat/Jenkinsfile.yml
@@ -88,3 +88,6 @@ stages:
     #            - "windows-7-32"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
\ No newline at end of file
diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml
index ec9a4ec57f09..1a9b6d1d4159 100644
--- a/x-pack/functionbeat/Jenkinsfile.yml
+++ b/x-pack/functionbeat/Jenkinsfile.yml
@@ -76,3 +76,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml
new file mode 100644
index 000000000000..4458db05e70b
--- /dev/null
+++ b/x-pack/heartbeat/Jenkinsfile.yml
@@ -0,0 +1,18 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^x-pack/heartbeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@xpack"             ## special token regarding the changeset for the xpack
+    comments:                  ## when PR comment contains any of those entries
+        - "/test x-pack/heartbeat"
+    labels:                    ## when PR labels matches any of those entries
+        - "x-pack-heartbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "x-pack-heartbeat"
+    tags: true                 ## for all the tags
+platform: "immutable && ubuntu-18" ## default label for all the stages
+stages:
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
index 90dcd4ca5602..b94ae1219cdc 100644
--- a/x-pack/metricbeat/Jenkinsfile.yml
+++ b/x-pack/metricbeat/Jenkinsfile.yml
@@ -79,3 +79,6 @@ stages:
 #        mage: "mage build unitTest"
 #        platforms:             ## override default labels in this specific stage.
 #            - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml
index 7cc39a8b5015..98ab5e2d72ed 100644
--- a/x-pack/packetbeat/Jenkinsfile.yml
+++ b/x-pack/packetbeat/Jenkinsfile.yml
@@ -82,3 +82,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml
index 0fcde2f326ce..49aa44448919 100644
--- a/x-pack/winlogbeat/Jenkinsfile.yml
+++ b/x-pack/winlogbeat/Jenkinsfile.yml
@@ -55,3 +55,8 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
+        platforms:             ## override default labels in this specific stage.
+            - "immutable && ubuntu-18"

From 6a0ea662999ec903268c4dcd12f2d8384d3eb2be Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 16:35:20 +0000
Subject: [PATCH 04/49] Fix command

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 25e973627ec2..255fbaeea4cb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -775,7 +775,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
     }
     if(args?.content?.containsKey('packaging-linux')) {
-      steps.packagingLinux(context: args.context, command: args.content.mage, directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
+      steps.packagingLinux(context: args.context, command: args.content.packaging-linux, directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
     }
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)

From e015b8e0d6b814d05318c94d7a4520708dbd853b Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 16:36:15 +0000
Subject: [PATCH 05/49] For testing purposes

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 255fbaeea4cb..843c583d77ef 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -768,21 +768,21 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   public run(Map args = [:]){
     def withModule = args.content.get('withModule', false)
     def isE2E = args.content.get('isE2E', false)
-    if(args?.content?.containsKey('make')) {
+    /*if(args?.content?.containsKey('make')) {
       steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
     }
     if(args?.content?.containsKey('mage')) {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
-    }
+    }*/
     if(args?.content?.containsKey('packaging-linux')) {
       steps.packagingLinux(context: args.context, command: args.content.packaging-linux, directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
-    }
+    }/*
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
     }
     if(args?.content?.containsKey('cloud')) {
       steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
-    }
+    } */
   }
 }
 

From 7f7b41e9caa0d3c2d09232d056a0f6fde31dd607 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 17:06:42 +0000
Subject: [PATCH 06/49] [CI] for testing purposes

---
 Jenkinsfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 843c583d77ef..e44b3f74b1c3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,6 +66,9 @@ pipeline {
     }
     stage('Lint'){
       options { skipDefaultCheckout() }
+      when {
+        expression { return false }
+      }
       environment {
         GOFLAGS = '-mod=readonly'
       }

From 8ad470735b6650c15ec7fcb60536fc11745a0edc Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 17:07:21 +0000
Subject: [PATCH 07/49] fix wrong syntax with a - in the field

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e44b3f74b1c3..e1c04fdc3f7b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -778,7 +778,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
     }*/
     if(args?.content?.containsKey('packaging-linux')) {
-      steps.packagingLinux(context: args.context, command: args.content.packaging-linux, directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
+      steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
     }/*
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)

From 121e52adbd01f67aa99f4bd778a4af9c9cbab61c Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Thu, 4 Feb 2021 20:46:45 +0000
Subject: [PATCH 08/49] [CI] fix credentials issue

---
 Jenkinsfile | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index e1c04fdc3f7b..f1bc0b99bdc5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -292,11 +292,8 @@ def e2e(Map args = [:]) {
   def tags = args.get('tags', '')
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
   dir('.e2e') {
-    gitCheckout(
-      branch: "master",  // TBC with the target branch if running on a PR basis.
-      repo: "https://github.com/elastic/e2e-testing.git",
-      credentialsId: "${JOB_GIT_CREDENTIALS}"
-    )
+    // TBC with the target branch if running on a PR basis.
+    gitCheckout(repo: "https://github.com/elastic/e2e-testing.git", branch: "master")
     try {
       if(isInstalled(tool: 'docker', flag: '--version')) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
@@ -312,7 +309,7 @@ def e2e(Map args = [:]) {
       error(e.toString())
     } finally {
       junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
-        archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
     }
   }
 }

From 5b61e057e8c6a2e223d654c146f514b94bf18037 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Fri, 5 Feb 2021 10:32:33 +0000
Subject: [PATCH 09/49] Use default git step (to avoid any kind of new env
 variables generated with gitCheckout)

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f1bc0b99bdc5..5af1cae4744d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -293,7 +293,7 @@ def e2e(Map args = [:]) {
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
   dir('.e2e') {
     // TBC with the target branch if running on a PR basis.
-    gitCheckout(repo: "https://github.com/elastic/e2e-testing.git", branch: "master")
+    git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
     try {
       if(isInstalled(tool: 'docker', flag: '--version')) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")

From 38b7a2e76179155e0b1e69dbbe5f21eabda2885f Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Fri, 5 Feb 2021 11:19:10 +0000
Subject: [PATCH 10/49] Cosmetic changes: use the right variable and docs

---
 Jenkinsfile | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5af1cae4744d..ccfbfd9a8648 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -9,7 +9,7 @@ pipeline {
     AWS_REGION = "${params.awsRegion}"
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
-    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_COMPOSE_VERSION = "1.21.0"
     DOCKER_REGISTRY = 'docker.elastic.co'
     JOB_GCS_BUCKET = 'beats-ci-temp'
@@ -287,6 +287,10 @@ def packagingLinux(Map args = [:]) {
   }
 }
 
+/**
+* This method runs the end 2 end testing in the same worker where the packages have been
+* generated, this should help to speed up the things
+*/
 def e2e(Map args = [:]) {
   def suite = args.get('suite', '')
   def tags = args.get('tags', '')
@@ -295,10 +299,10 @@ def e2e(Map args = [:]) {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
     try {
-      if(isInstalled(tool: 'docker', flag: '--version')) {
+      if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       }
-      retry(3){
+      retryWithSleep(retries: 3, seconds: 5){
         sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
       }
       filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
@@ -413,7 +417,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
     "USERPROFILE=${userProfile}"
   ]) {
     if(isDockerInstalled()) {
-      dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+      dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
     }
     dir("${env.BASE_DIR}") {
       installTools(args)

From 164830aa126aad71243d92fc956bf5f1295448ca Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Fri, 5 Feb 2021 12:08:06 +0000
Subject: [PATCH 11/49] Force GO_VERSION for the e2e to work

---
 Jenkinsfile | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ccfbfd9a8648..7f1122313b5d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -302,12 +302,15 @@ def e2e(Map args = [:]) {
       if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       }
-      retryWithSleep(retries: 3, seconds: 5){
-        sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
-      }
-      filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
-        // TBC with the suite to be used
-        sh script: """SUITE=metricbeat DEVELOPER_MODE=false TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE make -C e2e functional-test""", label: "Run functional tests for ${suite}:${tags}"
+      // Temporary fix to force the version instead picking up the env vironment variable.
+      withEnv(['GO_VERSION=1.14.12']) {
+        retryWithSleep(retries: 3, seconds: 5){
+          sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
+        }
+        filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
+          // TBC with the suite to be used
+          sh script: """SUITE=${suite} DEVELOPER_MODE=false TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE make -C e2e functional-test""", label: "Run functional tests for ${suite}:${tags}"
+        }
       }
     } catch(e) {
       error(e.toString())

From 1268e77da30eec499c71148de1c721d229390ae1 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Fri, 5 Feb 2021 14:17:03 +0000
Subject: [PATCH 12/49] Use dir for go

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7f1122313b5d..8ccb276033f2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -295,7 +295,7 @@ def e2e(Map args = [:]) {
   def suite = args.get('suite', '')
   def tags = args.get('tags', '')
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
-  dir('.e2e') {
+  dir("${env.WORKSPACE}/src/github.com/elastic") {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
     try {

From 0174527b90c264d3e71455b719ac1ae8db866b98 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Fri, 5 Feb 2021 15:12:23 +0000
Subject: [PATCH 13/49] Use checkout to skip clean folder with git

---
 Jenkinsfile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8ccb276033f2..b6e267fcbe17 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -297,7 +297,12 @@ def e2e(Map args = [:]) {
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
   dir("${env.WORKSPACE}/src/github.com/elastic") {
     // TBC with the target branch if running on a PR basis.
-    git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
+    // Use generic checkout to ensure the clean workspace is not executed, git step does it!
+    checkout(changelog: false,
+             poll: false,
+             scm: [$class: 'GitSCM', branches: [[name: 'master']],
+                   doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
+                   userRemoteConfigs: [[credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git']]])
     try {
       if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")

From 644e341411f5e31b3161b291bf773b12afbdac8b Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Fri, 5 Feb 2021 21:24:01 +0000
Subject: [PATCH 14/49] Revert "Use checkout to skip clean folder with git"

This reverts commit 0174527b90c264d3e71455b719ac1ae8db866b98.
---
 Jenkinsfile | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b6e267fcbe17..8ccb276033f2 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -297,12 +297,7 @@ def e2e(Map args = [:]) {
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
   dir("${env.WORKSPACE}/src/github.com/elastic") {
     // TBC with the target branch if running on a PR basis.
-    // Use generic checkout to ensure the clean workspace is not executed, git step does it!
-    checkout(changelog: false,
-             poll: false,
-             scm: [$class: 'GitSCM', branches: [[name: 'master']],
-                   doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [],
-                   userRemoteConfigs: [[credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git']]])
+    git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
     try {
       if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")

From 41a640734d85d5521e122800a154952753d57956 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Fri, 5 Feb 2021 21:24:52 +0000
Subject: [PATCH 15/49] Checkout in the folder

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 8ccb276033f2..c29d47d1167c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -295,7 +295,7 @@ def e2e(Map args = [:]) {
   def suite = args.get('suite', '')
   def tags = args.get('tags', '')
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
-  dir("${env.WORKSPACE}/src/github.com/elastic") {
+  dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
     try {

From d4e4cca19aa1d5bc5f5f990daafaa7077d0b2ed6 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Sat, 6 Feb 2021 11:19:51 +0000
Subject: [PATCH 16/49] Return boolean

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index c29d47d1167c..fb67d1530e31 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -741,7 +741,7 @@ def dumpVariables(){
 def isDockerInstalled(){
   if (isUnix()) {
     // TODO: some issues with macosx if(isInstalled(tool: 'docker', flag: '--version')) {
-    return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true)
+    return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) == 0
   } else {
     return false
   }

From ecfd31be98460bb9de19a2420a5fe63c5b652515 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Sun, 7 Feb 2021 10:12:18 +0000
Subject: [PATCH 17/49] Use junit reporting

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index fb67d1530e31..f783d398f8b0 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -309,14 +309,14 @@ def e2e(Map args = [:]) {
         }
         filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
           // TBC with the suite to be used
-          sh script: """SUITE=${suite} DEVELOPER_MODE=false TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE make -C e2e functional-test""", label: "Run functional tests for ${suite}:${tags}"
+          sh script: """FORMAT=junit:TEST-${suite}.xml SUITE=${suite} DEVELOPER_MODE=false TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE make -C e2e functional-test""", label: "Run functional tests for ${suite}:${tags}"
         }
       }
     } catch(e) {
       error(e.toString())
     } finally {
-      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
-      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "TEST-*.xml")
+      archiveArtifacts allowEmptyArchive: true, artifacts: "TEST-*.xml"
     }
   }
 }

From 3f790307e7876e7561e41fedf9119d4279dab205 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Sun, 7 Feb 2021 10:21:48 +0000
Subject: [PATCH 18/49] Test metricbeat script

---
 Jenkinsfile | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f783d398f8b0..65469b9f97dc 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -297,7 +297,7 @@ def e2e(Map args = [:]) {
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
-    git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
+    git(branch: 'feature/metricbeat-goal', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
     try {
       if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
@@ -307,16 +307,15 @@ def e2e(Map args = [:]) {
         retryWithSleep(retries: 3, seconds: 5){
           sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
         }
-        filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
-          // TBC with the suite to be used
-          sh script: """FORMAT=junit:TEST-${suite}.xml SUITE=${suite} DEVELOPER_MODE=false TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE make -C e2e functional-test""", label: "Run functional tests for ${suite}:${tags}"
+        filebeat(output: "docker_logs_${suite}.log", workdir: "${env.WORKSPACE}"){
+          sh script: ".ci/scripts/${suite}-test.sh", label: "Run functional tests for ${suite}"
         }
       }
     } catch(e) {
       error(e.toString())
     } finally {
-      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "TEST-*.xml")
-      archiveArtifacts allowEmptyArchive: true, artifacts: "TEST-*.xml"
+      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
+      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
     }
   }
 }

From 481c3cf29201bbe25c0c7128aee0a35cd6cdf8f3 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Sun, 7 Feb 2021 17:15:55 +0000
Subject: [PATCH 19/49] Delegate test preparation to the e2e scripts

---
 Jenkinsfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 65469b9f97dc..ea5d239dab39 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -304,9 +304,6 @@ def e2e(Map args = [:]) {
       }
       // Temporary fix to force the version instead picking up the env vironment variable.
       withEnv(['GO_VERSION=1.14.12']) {
-        retryWithSleep(retries: 3, seconds: 5){
-          sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
-        }
         filebeat(output: "docker_logs_${suite}.log", workdir: "${env.WORKSPACE}"){
           sh script: ".ci/scripts/${suite}-test.sh", label: "Run functional tests for ${suite}"
         }

From 872bcbaacf90efa35250e75ff97ca905633d10bb Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Sun, 7 Feb 2021 17:16:53 +0000
Subject: [PATCH 20/49] Prepare Go version dynamically

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ea5d239dab39..84ebb149da4b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -302,8 +302,8 @@ def e2e(Map args = [:]) {
       if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       }
-      // Temporary fix to force the version instead picking up the env vironment variable.
-      withEnv(['GO_VERSION=1.14.12']) {
+      def goVersionForE2E = readFile(".go-version").trim()
+      withEnv(["GO_VERSION=${goVersionForE2E}"]) {
         filebeat(output: "docker_logs_${suite}.log", workdir: "${env.WORKSPACE}"){
           sh script: ".ci/scripts/${suite}-test.sh", label: "Run functional tests for ${suite}"
         }

From 1222ca4af2e37f052564533afa85474b2d86fd7d Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Mon, 8 Feb 2021 16:36:02 +0000
Subject: [PATCH 21/49] Use entrypoint

---
 Jenkinsfile                          | 20 +++++++++++---------
 auditbeat/Jenkinsfile.yml            |  3 ++-
 filebeat/Jenkinsfile.yml             |  3 ++-
 heartbeat/Jenkinsfile.yml            |  3 ++-
 journalbeat/Jenkinsfile.yml          |  3 ++-
 metricbeat/Jenkinsfile.yml           |  4 +++-
 packetbeat/Jenkinsfile.yml           |  3 ++-
 winlogbeat/Jenkinsfile.yml           |  3 ++-
 x-pack/auditbeat/Jenkinsfile.yml     |  3 ++-
 x-pack/dockerlogbeat/Jenkinsfile.yml |  3 ++-
 x-pack/elastic-agent/Jenkinsfile.yml |  3 ++-
 x-pack/filebeat/Jenkinsfile.yml      |  3 ++-
 x-pack/functionbeat/Jenkinsfile.yml  |  3 ++-
 x-pack/heartbeat/Jenkinsfile.yml     |  3 ++-
 x-pack/metricbeat/Jenkinsfile.yml    |  3 ++-
 x-pack/packetbeat/Jenkinsfile.yml    |  3 ++-
 x-pack/winlogbeat/Jenkinsfile.yml    |  3 ++-
 17 files changed, 44 insertions(+), 25 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 84ebb149da4b..3d88a9ed2740 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -292,9 +292,10 @@ def packagingLinux(Map args = [:]) {
 * generated, this should help to speed up the things
 */
 def e2e(Map args = [:]) {
-  def suite = args.get('suite', '')
-  def tags = args.get('tags', '')
+  def enabled = args.e2e?.get('enabled', false)
+  def entrypoint = args.e2e?.get('entrypoint')
   def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
+  if (!enabled) { return }
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'feature/metricbeat-goal', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
@@ -302,10 +303,11 @@ def e2e(Map args = [:]) {
       if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
       }
-      def goVersionForE2E = readFile(".go-version").trim()
-      withEnv(["GO_VERSION=${goVersionForE2E}"]) {
-        filebeat(output: "docker_logs_${suite}.log", workdir: "${env.WORKSPACE}"){
-          sh script: ".ci/scripts/${suite}-test.sh", label: "Run functional tests for ${suite}"
+      def goVersionForE2E = readFile('.go-version').trim()
+      withEnv(["GO_VERSION=${goVersionForE2E}",
+               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}"]) {
+        filebeat(output: "docker_logs_${entrypoint}.log", workdir: "${env.WORKSPACE}"){
+          sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
         }
       }
     } catch(e) {
@@ -338,8 +340,9 @@ def target(Map args = [:]) {
         dir(isMage ? directory : '') {
           cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
         }
+        // If package then upload artifacts
         if(isE2E) {
-          e2e(suite: directory)
+          e2e(args)
         }
       }
     }
@@ -770,7 +773,6 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   }
   public run(Map args = [:]){
     def withModule = args.content.get('withModule', false)
-    def isE2E = args.content.get('isE2E', false)
     /*if(args?.content?.containsKey('make')) {
       steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
     }
@@ -778,7 +780,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
     }*/
     if(args?.content?.containsKey('packaging-linux')) {
-      steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
+      steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'))
     }/*
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
index 52c3bf9f02ec..346145606b50 100644
--- a/auditbeat/Jenkinsfile.yml
+++ b/auditbeat/Jenkinsfile.yml
@@ -85,4 +85,5 @@ stages:
     #        tags: true         ## for all the tags
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
index e9e4bf55013c..109474258283 100644
--- a/filebeat/Jenkinsfile.yml
+++ b/filebeat/Jenkinsfile.yml
@@ -83,4 +83,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
index 4693c1ae3985..8c1bbeaa574d 100644
--- a/heartbeat/Jenkinsfile.yml
+++ b/heartbeat/Jenkinsfile.yml
@@ -82,4 +82,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml
index 37b4ab233089..0336f113162e 100644
--- a/journalbeat/Jenkinsfile.yml
+++ b/journalbeat/Jenkinsfile.yml
@@ -35,4 +35,5 @@ stages:
         mage: "mage build unitTest"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
index ed1edad19720..3204b7dd2888 100644
--- a/metricbeat/Jenkinsfile.yml
+++ b/metricbeat/Jenkinsfile.yml
@@ -77,4 +77,6 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: true
+        e2e:
+            enabled: true
+            entrypoint: 'metricbeat-test.sh'
diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
index f00a129d4d7e..3205b791b791 100644
--- a/packetbeat/Jenkinsfile.yml
+++ b/packetbeat/Jenkinsfile.yml
@@ -83,4 +83,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml
index 047cda73007d..a66ffcf1c732 100644
--- a/winlogbeat/Jenkinsfile.yml
+++ b/winlogbeat/Jenkinsfile.yml
@@ -53,4 +53,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
index c594fb5990c8..7a1edec5dd27 100644
--- a/x-pack/auditbeat/Jenkinsfile.yml
+++ b/x-pack/auditbeat/Jenkinsfile.yml
@@ -83,4 +83,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml
index 12db0b0c3efe..ee9a4b06aca5 100644
--- a/x-pack/dockerlogbeat/Jenkinsfile.yml
+++ b/x-pack/dockerlogbeat/Jenkinsfile.yml
@@ -23,4 +23,5 @@ stages:
         withModule: true       ## run the ITs only if the changeset affects a specific module.
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml
index b6daada774b3..9a30d836391b 100644
--- a/x-pack/elastic-agent/Jenkinsfile.yml
+++ b/x-pack/elastic-agent/Jenkinsfile.yml
@@ -87,4 +87,5 @@ stages:
     #        tags: true         ## for all the tags
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
index 516100eb038f..5985900fcdad 100644
--- a/x-pack/filebeat/Jenkinsfile.yml
+++ b/x-pack/filebeat/Jenkinsfile.yml
@@ -90,4 +90,5 @@ stages:
     #        tags: true         ## for all the tags
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
\ No newline at end of file
+        e2e:
+            enabled: false
\ No newline at end of file
diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml
index 1a9b6d1d4159..c6cb19ddcf6c 100644
--- a/x-pack/functionbeat/Jenkinsfile.yml
+++ b/x-pack/functionbeat/Jenkinsfile.yml
@@ -78,4 +78,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml
index 4458db05e70b..b715d109a12c 100644
--- a/x-pack/heartbeat/Jenkinsfile.yml
+++ b/x-pack/heartbeat/Jenkinsfile.yml
@@ -15,4 +15,5 @@ platform: "immutable && ubuntu-18" ## default label for all the stages
 stages:
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
index e4f97dbd6b2d..f9416fd5efcb 100644
--- a/x-pack/metricbeat/Jenkinsfile.yml
+++ b/x-pack/metricbeat/Jenkinsfile.yml
@@ -81,4 +81,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml
index 98ab5e2d72ed..dcefc94c6694 100644
--- a/x-pack/packetbeat/Jenkinsfile.yml
+++ b/x-pack/packetbeat/Jenkinsfile.yml
@@ -84,4 +84,5 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml
index 49aa44448919..8b03fed80a3e 100644
--- a/x-pack/winlogbeat/Jenkinsfile.yml
+++ b/x-pack/winlogbeat/Jenkinsfile.yml
@@ -57,6 +57,7 @@ stages:
             - "windows-7-32-bit"
     packaging-linux:
         packaging-linux: "mage package"
-        isE2E: false
+        e2e:
+            enabled: false
         platforms:             ## override default labels in this specific stage.
             - "immutable && ubuntu-18"

From 0cd5c081912e388920e9cdb9acf3556fe7f610f6 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Mon, 8 Feb 2021 16:46:06 +0000
Subject: [PATCH 22/49] Publish packages

---
 Jenkinsfile | 42 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 39 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3d88a9ed2740..9c41c21d109b 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -287,6 +287,42 @@ def packagingLinux(Map args = [:]) {
   }
 }
 
+def publishPackages(baseDir){
+  def bucketUri = "gs://beats-ci-artifacts/snapshots"
+  if (isPR()) {
+    bucketUri = "gs://beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}"
+  }
+  def beatsFolderName = getBeatsName(baseDir)
+  uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+
+  // Copy those files to another location with the sha commit to test them
+  // afterward.
+  bucketUri = "gs://beats-ci-artifacts/commits/${env.GIT_BASE_COMMIT}"
+  uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+}
+
+def uploadPackages(bucketUri, baseDir){
+  googleStorageUpload(bucket: bucketUri,
+    credentialsId: "${JOB_GCS_CREDENTIALS}",
+    pathPrefix: "${baseDir}/build/distributions/",
+    pattern: "${baseDir}/build/distributions/**/*",
+    sharedPublicly: true,
+    showInline: true
+  )
+}
+
+/**
+* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
+* therefore the storage bucket in GCP should follow the same folder structure.
+* This is required by https://github.com/elastic/beats-tester
+* e.g.
+* baseDir=name -> return name
+* baseDir=name1/name2/name3-> return name2
+*/
+def getBeatsName(baseDir) {
+  return baseDir.replace('x-pack/', '')
+}
+
 /**
 * This method runs the end 2 end testing in the same worker where the packages have been
 * generated, this should help to speed up the things
@@ -325,12 +361,12 @@ def e2e(Map args = [:]) {
 *  - mage then the dir(location) is required, aka by enabling isMage: true.
 */
 def target(Map args = [:]) {
-  def context = args.context
   def command = args.command
+  def context = args.context
   def directory = args.get('directory', '')
   def withModule = args.get('withModule', false)
   def isMage = args.get('isMage', false)
-  def isE2E = args.get('isE2E', false)
+  def isE2E = args.e2e?.get('enabled', false)
   withNode(args.label) {
     withGithubNotify(context: "${context}") {
       withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
@@ -340,9 +376,9 @@ def target(Map args = [:]) {
         dir(isMage ? directory : '') {
           cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
         }
-        // If package then upload artifacts
         if(isE2E) {
           e2e(args)
+          publishPackages("${directory}")
         }
       }
     }

From 22997575a84bd9713c684b74996b9cf7d2a15d01 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Mon, 8 Feb 2021 17:38:20 +0000
Subject: [PATCH 23/49] Push CI docker images

---
 Jenkinsfile | 110 ++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 102 insertions(+), 8 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9c41c21d109b..298dd73c604c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -287,30 +287,123 @@ def packagingLinux(Map args = [:]) {
   }
 }
 
-def publishPackages(baseDir){
+
+/**
+* Upload the packages to their snapshot or pull request buckets
+* @param beatsFolder beats folder
+*/
+def publishPackages(beatsFolder){
   def bucketUri = "gs://beats-ci-artifacts/snapshots"
   if (isPR()) {
     bucketUri = "gs://beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}"
   }
-  def beatsFolderName = getBeatsName(baseDir)
-  uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+  def beatsFolderName = getBeatsName(beatsFolder)
+  uploadPackages("${bucketUri}/${beatsFolderName}", beatsFolder)
 
   // Copy those files to another location with the sha commit to test them
   // afterward.
   bucketUri = "gs://beats-ci-artifacts/commits/${env.GIT_BASE_COMMIT}"
-  uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+  uploadPackages("${bucketUri}/${beatsFolderName}", beatsFolder)
 }
 
-def uploadPackages(bucketUri, baseDir){
+/**
+* Upload the distribution files to google cloud.
+* TODO: There is a known issue with Google Storage plugin.
+* @param bucketUri the buckets URI.
+* @param beatsFolder the beats folder.
+*/
+def uploadPackages(bucketUri, beatsFolder){
   googleStorageUpload(bucket: bucketUri,
     credentialsId: "${JOB_GCS_CREDENTIALS}",
-    pathPrefix: "${baseDir}/build/distributions/",
-    pattern: "${baseDir}/build/distributions/**/*",
+    pathPrefix: "${beatsFolder}/build/distributions/",
+    pattern: "${beatsFolder}/build/distributions/**/*",
     sharedPublicly: true,
     showInline: true
   )
 }
 
+/**
+* Push the docker images for the given beat.
+* @param beatsFolder beats folder
+*/
+def pushCIDockerImages(beatsFolder){
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    if (beatsFolder.endsWith('auditbeat')) {
+      tagAndPush('auditbeat')
+    } else if (beatsFolder.endsWith('filebeat')) {
+      tagAndPush('filebeat')
+    } else if (beatsFolder.endsWith('heartbeat')) {
+      tagAndPush('heartbeat')
+    } else if ("${beatsFolder}" == "journalbeat"){
+      tagAndPush('journalbeat')
+    } else if (beatsFolder.endsWith('metricbeat')) {
+      tagAndPush('metricbeat')
+    } else if ("${beatsFolder}" == "packetbeat"){
+      tagAndPush('packetbeat')
+    } else if ("${beatsFolder}" == "x-pack/elastic-agent") {
+      tagAndPush('elastic-agent')
+    }
+  }
+}
+
+/**
+* Tag and push all the docker images for the given beat.
+* @param beatName name of the Beat
+*/
+def tagAndPush(beatName){
+  def libbetaVer = env.VERSION
+  def aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
+
+  libbetaVer += "-SNAPSHOT"
+  aliasVersion += "-SNAPSHOT"
+
+  def tagName = "${libbetaVer}"
+  if (isPR()) {
+    tagName = "pr-${env.CHANGE_ID}"
+  }
+
+  // supported image flavours
+  def variants = ["", "-oss", "-ubi8"]
+  variants.each { variant ->
+    doTagAndPush(beatName, variant, libbetaVer, tagName)
+    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
+
+    if (!isPR() && aliasVersion != "") {
+      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    }
+  }
+}
+
+/**
+* Tag and push the given sourceTag docker image with the tag name targetTag.
+* @param beatName name of the Beat
+* @param variant name of the variant used to build the docker image name
+* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
+* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
+*/
+def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
+
+  def iterations = 0
+  retryWithSleep(retries: 3, seconds: 5, backoff: true) {
+    iterations++
+    def status = sh(label: "Change tag and push ${targetName}", script: """
+      if docker image inspect "${sourceName}" &> /dev/null ; then
+        docker tag ${sourceName} ${targetName}
+        docker push ${targetName}
+      else
+        echo 'docker image ${sourceName} does not exist'
+      fi
+    """, returnStatus: true)
+    if ( status > 0 && iterations < 3) {
+      error("tag and push failed for ${beatName}, retry")
+    } else if ( status > 0 ) {
+      log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
+    }
+  }
+}
+
 /**
 * There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
 * therefore the storage bucket in GCP should follow the same folder structure.
@@ -378,8 +471,9 @@ def target(Map args = [:]) {
         }
         if(isE2E) {
           e2e(args)
-          publishPackages("${directory}")
         }
+        publishPackages("${directory}")
+        pushCIDockerImages("${directory}")
       }
     }
   }

From 58e5bc571e60df4f9f8e0525796d197ae1440812 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Mon, 8 Feb 2021 17:51:30 +0000
Subject: [PATCH 24/49] Use e2e master branch

---
 Jenkinsfile | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 298dd73c604c..0909740aac6e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -423,11 +423,10 @@ def getBeatsName(baseDir) {
 def e2e(Map args = [:]) {
   def enabled = args.e2e?.get('enabled', false)
   def entrypoint = args.e2e?.get('entrypoint')
-  def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
   if (!enabled) { return }
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
-    git(branch: 'feature/metricbeat-goal', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
+    git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
     try {
       if(isDockerInstalled()) {
         dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")

From 5c9d7ea36ed636efe7974142356b3f1957fe59cf Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 10:27:03 +0000
Subject: [PATCH 25/49] Disable other pipelines

---
 Jenkinsfile.yml | 40 ++++++++++++++++++++--------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
index 284000da86fe..7591e577f633 100644
--- a/Jenkinsfile.yml
+++ b/Jenkinsfile.yml
@@ -1,25 +1,25 @@
 projects:
-    - "auditbeat"
-    - "deploy/kubernetes"
-    - "filebeat"
-    - "generator"
-    - "heartbeat"
-    - "journalbeat"
-    - "libbeat"
+    #- "auditbeat"
+    #- "deploy/kubernetes"
+    #- "filebeat"
+    #- "generator"
+    #- "heartbeat"
+    #- "journalbeat"
+    #- "libbeat"
     - "metricbeat"
-    - "packetbeat"
-    - "winlogbeat"
-    - "x-pack/auditbeat"
-    - "x-pack/dockerlogbeat"
-    - "x-pack/elastic-agent"
-    - "x-pack/filebeat"
-    - "x-pack/functionbeat"
-    - "x-pack/heartbeat"
-    - "x-pack/libbeat"
-    - "x-pack/metricbeat"
-    - "x-pack/packetbeat"
-    - "x-pack/winlogbeat"
-    - "dev-tools"
+    #- "packetbeat"
+    #- "winlogbeat"
+    #- "x-pack/auditbeat"
+    #- "x-pack/dockerlogbeat"
+    #- "x-pack/elastic-agent"
+    #- "x-pack/filebeat"
+    #- "x-pack/functionbeat"
+    #- "x-pack/heartbeat"
+    #- "x-pack/libbeat"
+    #- "x-pack/metricbeat"
+    #- "x-pack/packetbeat"
+    #- "x-pack/winlogbeat"
+    #- "dev-tools"
     ##- "x-pack/journalbeat"  It is not yet in the 1.0 pipeline.
 
 ## Changeset macros that are defined here and used in each specific 2.0 pipeline.

From 85260d807ca170839442340a22eb1d0bdd3837d9 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 10:30:53 +0000
Subject: [PATCH 26/49] Add debug mode

---
 Jenkinsfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 0909740aac6e..a8c17cfc850e 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -433,7 +433,8 @@ def e2e(Map args = [:]) {
       }
       def goVersionForE2E = readFile('.go-version').trim()
       withEnv(["GO_VERSION=${goVersionForE2E}",
-               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}"]) {
+               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
+               "OP_LOG_LEVEL=DEBUG"]) {
         filebeat(output: "docker_logs_${entrypoint}.log", workdir: "${env.WORKSPACE}"){
           sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
         }

From d0371d4971fae64e4bfab09528cc8a77fa4cc86a Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 11:06:56 +0000
Subject: [PATCH 27/49] Rename LOG level variable

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index a8c17cfc850e..5277dab9aef5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -434,7 +434,7 @@ def e2e(Map args = [:]) {
       def goVersionForE2E = readFile('.go-version').trim()
       withEnv(["GO_VERSION=${goVersionForE2E}",
                "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
-               "OP_LOG_LEVEL=DEBUG"]) {
+               "LOG_LEVEL=TRACE"]) {
         filebeat(output: "docker_logs_${entrypoint}.log", workdir: "${env.WORKSPACE}"){
           sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
         }

From 96ff9959061029200c07626fd528bf693d2ec86a Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 12:02:29 +0000
Subject: [PATCH 28/49] Avoid using the workdir

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5277dab9aef5..b3ef844ff862 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -435,7 +435,7 @@ def e2e(Map args = [:]) {
       withEnv(["GO_VERSION=${goVersionForE2E}",
                "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
                "LOG_LEVEL=TRACE"]) {
-        filebeat(output: "docker_logs_${entrypoint}.log", workdir: "${env.WORKSPACE}"){
+        filebeat(output: "docker_logs_${entrypoint}.log"){
           sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
         }
       }

From 1862d0341c32e8b522327a13a2f3d8c2938749e0 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 13:14:05 +0000
Subject: [PATCH 29/49] For debugging purposes

---
 Jenkinsfile | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index b3ef844ff862..b6ebc17492da 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -423,7 +423,10 @@ def getBeatsName(baseDir) {
 def e2e(Map args = [:]) {
   def enabled = args.e2e?.get('enabled', false)
   def entrypoint = args.e2e?.get('entrypoint')
+  def directory = args.get('directory', '')
   if (!enabled) { return }
+  sh "ls -ltrah ${env.WORKSPACE}/${env.BASE_DIR}/${directory}/build/distributions || true"
+  sh 'find . -name "*tar.gz" -ls || true'
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
@@ -440,6 +443,8 @@ def e2e(Map args = [:]) {
         }
       }
     } catch(e) {
+      sh 'ls -ltra'
+      sh 'cat /var/lib/jenkins/workspace/Beats_beats_PR-23854/src/github.com/elastic/e2e-testing/*.log || true'
       error(e.toString())
     } finally {
       junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")

From 3ac5cb24e8722fa0ea12a727d9ed1084731a844d Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 14:02:44 +0000
Subject: [PATCH 30/49] Revert "For debugging purposes"

This reverts commit 1862d0341c32e8b522327a13a2f3d8c2938749e0.
---
 Jenkinsfile | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b6ebc17492da..b3ef844ff862 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -423,10 +423,7 @@ def getBeatsName(baseDir) {
 def e2e(Map args = [:]) {
   def enabled = args.e2e?.get('enabled', false)
   def entrypoint = args.e2e?.get('entrypoint')
-  def directory = args.get('directory', '')
   if (!enabled) { return }
-  sh "ls -ltrah ${env.WORKSPACE}/${env.BASE_DIR}/${directory}/build/distributions || true"
-  sh 'find . -name "*tar.gz" -ls || true'
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
@@ -443,8 +440,6 @@ def e2e(Map args = [:]) {
         }
       }
     } catch(e) {
-      sh 'ls -ltra'
-      sh 'cat /var/lib/jenkins/workspace/Beats_beats_PR-23854/src/github.com/elastic/e2e-testing/*.log || true'
       error(e.toString())
     } finally {
       junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")

From 3e073bff4402fe81c80e6ab8753b03665df5db59 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 14:03:56 +0000
Subject: [PATCH 31/49] Revert "Avoid using the workdir"

This reverts commit 96ff9959061029200c07626fd528bf693d2ec86a.
---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b3ef844ff862..5277dab9aef5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -435,7 +435,7 @@ def e2e(Map args = [:]) {
       withEnv(["GO_VERSION=${goVersionForE2E}",
                "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
                "LOG_LEVEL=TRACE"]) {
-        filebeat(output: "docker_logs_${entrypoint}.log"){
+        filebeat(output: "docker_logs_${entrypoint}.log", workdir: "${env.WORKSPACE}"){
           sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
         }
       }

From 05af27a48ff6ba7788b6ae5cf3a88b06ef8ab593 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 14:05:59 +0000
Subject: [PATCH 32/49] Throw error and archive docker logs

---
 Jenkinsfile | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 5277dab9aef5..b6de6db4d66d 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -423,6 +423,7 @@ def getBeatsName(baseDir) {
 def e2e(Map args = [:]) {
   def enabled = args.e2e?.get('enabled', false)
   def entrypoint = args.e2e?.get('entrypoint')
+  def dockerLogFile = "docker_logs_${entrypoint}.log"
   if (!enabled) { return }
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
@@ -435,15 +436,13 @@ def e2e(Map args = [:]) {
       withEnv(["GO_VERSION=${goVersionForE2E}",
                "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
                "LOG_LEVEL=TRACE"]) {
-        filebeat(output: "docker_logs_${entrypoint}.log", workdir: "${env.WORKSPACE}"){
+        filebeat(output: dockerLogFile, workdir: "${env.WORKSPACE}"){
           sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
         }
       }
-    } catch(e) {
-      error(e.toString())
     } finally {
       junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
-      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml, ${env.WORKSPACE}/${dockerLogFile}.log"
     }
   }
 }

From 6da223e5a1efdf7168b5a478e3644fcca8aa11d8 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 14:08:44 +0000
Subject: [PATCH 33/49] Use x-pack e2e

---
 Jenkinsfile.yml                   | 2 +-
 x-pack/metricbeat/Jenkinsfile.yml | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
index 7591e577f633..587bda77737b 100644
--- a/Jenkinsfile.yml
+++ b/Jenkinsfile.yml
@@ -16,7 +16,7 @@ projects:
     #- "x-pack/functionbeat"
     #- "x-pack/heartbeat"
     #- "x-pack/libbeat"
-    #- "x-pack/metricbeat"
+    - "x-pack/metricbeat"
     #- "x-pack/packetbeat"
     #- "x-pack/winlogbeat"
     #- "dev-tools"
diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
index f9416fd5efcb..08aaa1d6fc0f 100644
--- a/x-pack/metricbeat/Jenkinsfile.yml
+++ b/x-pack/metricbeat/Jenkinsfile.yml
@@ -83,3 +83,4 @@ stages:
         packaging-linux: "mage package"
         e2e:
             enabled: false
+            entrypoint: 'metricbeat-test.sh'

From c0b71301ac03da474c4d18a42cd63e252c51b18b Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 14:31:53 +0000
Subject: [PATCH 34/49] Avoid using the workdir since the archive does not work
 with full paths

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index b6de6db4d66d..d3129f3b501c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -436,13 +436,13 @@ def e2e(Map args = [:]) {
       withEnv(["GO_VERSION=${goVersionForE2E}",
                "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
                "LOG_LEVEL=TRACE"]) {
-        filebeat(output: dockerLogFile, workdir: "${env.WORKSPACE}"){
+        filebeat(output: dockerLogFile){
           sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
         }
       }
     } finally {
       junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
-      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml, ${env.WORKSPACE}/${dockerLogFile}.log"
+      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
     }
   }
 }

From 0d819cddbe62439daad49c03168cbeb5bb7f3ce1 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 14:36:08 +0000
Subject: [PATCH 35/49] Move away from try/catch

---
 Jenkinsfile | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index d3129f3b501c..4dccf3dd9098 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -428,21 +428,24 @@ def e2e(Map args = [:]) {
   dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
     // TBC with the target branch if running on a PR basis.
     git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
-    try {
-      if(isDockerInstalled()) {
-        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-      }
-      def goVersionForE2E = readFile('.go-version').trim()
-      withEnv(["GO_VERSION=${goVersionForE2E}",
-               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
-               "LOG_LEVEL=TRACE"]) {
-        filebeat(output: dockerLogFile){
-          sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
-        }
+    if(isDockerInstalled()) {
+      dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+    }
+    def goVersionForE2E = readFile('.go-version').trim()
+    withEnv(["GO_VERSION=${goVersionForE2E}",
+              "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}",
+              "LOG_LEVEL=TRACE"]) {
+      def status = 0
+      filebeat(output: dockerLogFile){
+        status = sh(script: ".ci/scripts/${entrypoint}",
+                    label: "Run functional tests ${entrypoint}",
+                    returnStatus: true)
       }
-    } finally {
       junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
       archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+      if (status != 0) {
+        error("ERROR: functional tests for ${args?.directory?.trim()} has failed. See the test report and ${dockerLogFile}.")
+      }
     }
   }
 }

From 92eea8fc225447f52fe72ab3a544818b84efe5b2 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 15:45:48 +0000
Subject: [PATCH 36/49] Add e2e

---
 x-pack/metricbeat/Jenkinsfile.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
index 08aaa1d6fc0f..df2e0b89b2aa 100644
--- a/x-pack/metricbeat/Jenkinsfile.yml
+++ b/x-pack/metricbeat/Jenkinsfile.yml
@@ -82,5 +82,5 @@ stages:
     packaging-linux:
         packaging-linux: "mage package"
         e2e:
-            enabled: false
+            enabled: true
             entrypoint: 'metricbeat-test.sh'

From 896703d7543616a4fdc1b78cba66cf7165f7bccd Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 15:46:06 +0000
Subject: [PATCH 37/49] Rename env variable

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 4dccf3dd9098..9526a63b53df 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -76,7 +76,7 @@ pipeline {
         withGithubNotify(context: "Lint") {
           withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
-            setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
+            setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
             whenTrue(env.ONLY_DOCS == 'true') {
               cmd(label: "make check", script: "make check")
             }
@@ -150,7 +150,7 @@ pipeline {
     success {
       writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline
 COMMIT=${env.GIT_BASE_COMMIT}
-VERSION=${env.VERSION}-SNAPSHOT""")
+VERSION=${env.BEAT_VERSION}-SNAPSHOT""")
       archiveArtifacts artifacts: 'packaging.properties'
     }
     cleanup {
@@ -351,7 +351,7 @@ def pushCIDockerImages(beatsFolder){
 * @param beatName name of the Beat
 */
 def tagAndPush(beatName){
-  def libbetaVer = env.VERSION
+  def libbetaVer = env.BEAT_VERSION
   def aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
 
   libbetaVer += "-SNAPSHOT"

From 6380624082aa2c07d405505c7080eab0db95ccd4 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 17:47:35 +0000
Subject: [PATCH 38/49] Disable e2e for oss metricbeats

---
 metricbeat/Jenkinsfile.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
index 3204b7dd2888..70874ab12779 100644
--- a/metricbeat/Jenkinsfile.yml
+++ b/metricbeat/Jenkinsfile.yml
@@ -78,5 +78,4 @@ stages:
     packaging-linux:
         packaging-linux: "mage package"
         e2e:
-            enabled: true
-            entrypoint: 'metricbeat-test.sh'
+            enabled: false     ## e2e is enabled only for x-pack beats

From a70a3805a751ceb635130071feb3633e1e1636ca Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 20:27:47 +0000
Subject: [PATCH 39/49] Revert "[CI] for testing purposes"

This reverts commit 7f7b41e9caa0d3c2d09232d056a0f6fde31dd607.
---
 Jenkinsfile | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 9526a63b53df..af09f81b2251 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -66,9 +66,6 @@ pipeline {
     }
     stage('Lint'){
       options { skipDefaultCheckout() }
-      when {
-        expression { return false }
-      }
       environment {
         GOFLAGS = '-mod=readonly'
       }

From 984f533a2e5201f54f34af6e21c8844bd07432f2 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 20:28:47 +0000
Subject: [PATCH 40/49] Rename env variable

---
 Jenkinsfile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index af09f81b2251..61dd3db371d5 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -73,7 +73,7 @@ pipeline {
         withGithubNotify(context: "Lint") {
           withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
-            setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
+            setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
             whenTrue(env.ONLY_DOCS == 'true') {
               cmd(label: "make check", script: "make check")
             }
@@ -147,7 +147,7 @@ pipeline {
     success {
       writeFile(file: 'packaging.properties', text: """## To be consumed by the packaging pipeline
 COMMIT=${env.GIT_BASE_COMMIT}
-VERSION=${env.BEAT_VERSION}-SNAPSHOT""")
+VERSION=${env.VERSION}-SNAPSHOT""")
       archiveArtifacts artifacts: 'packaging.properties'
     }
     cleanup {
@@ -348,7 +348,7 @@ def pushCIDockerImages(beatsFolder){
 * @param beatName name of the Beat
 */
 def tagAndPush(beatName){
-  def libbetaVer = env.BEAT_VERSION
+  def libbetaVer = env.VERSION
   def aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
 
   libbetaVer += "-SNAPSHOT"

From 7522126967e21e5a3ebb9b9d6e7c5ebf9552281c Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 20:30:09 +0000
Subject: [PATCH 41/49] [chore] for testing purposes, disable linting

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 61dd3db371d5..3bfc01e8df05 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -74,7 +74,7 @@ pipeline {
           withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
             setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
-            whenTrue(env.ONLY_DOCS == 'true') {
+            /*whenTrue(env.ONLY_DOCS == 'true') {
               cmd(label: "make check", script: "make check")
             }
             whenTrue(env.ONLY_DOCS == 'false') {
@@ -82,7 +82,7 @@ pipeline {
               cmd(label: "make check-go", script: "make check-go")
               cmd(label: "make notice", script: "make notice")
               cmd(label: "Check for changes", script: "make check-no-changes")
-            }
+            }*/
           }
         }
       }

From 09c12ab81642eb21827955ad17ecb2bccbe1ea49 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 20:36:20 +0000
Subject: [PATCH 42/49] Upload artifacts earlier

---
 Jenkinsfile | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 3bfc01e8df05..ba2c6519c31c 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -459,6 +459,7 @@ def target(Map args = [:]) {
   def withModule = args.get('withModule', false)
   def isMage = args.get('isMage', false)
   def isE2E = args.e2e?.get('enabled', false)
+  def isPackaging = args.get('package', false)
   withNode(args.label) {
     withGithubNotify(context: "${context}") {
       withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
@@ -468,11 +469,19 @@ def target(Map args = [:]) {
         dir(isMage ? directory : '') {
           cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
         }
+        // TODO:
+        // Packaging should happen only after the e2e?
+        if (isPackaging) {
+          publishPackages("${directory}")
+        }
         if(isE2E) {
           e2e(args)
         }
-        publishPackages("${directory}")
-        pushCIDockerImages("${directory}")
+        // TODO:
+        // push docker images should happen only after the e2e?
+        if (isPackaging) {
+          pushCIDockerImages("${directory}")
+        }
       }
     }
   }
@@ -909,7 +918,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
     }*/
     if(args?.content?.containsKey('packaging-linux')) {
-      steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'))
+      steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true)
     }/*
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)

From 97e740900161101efc61cad8cb95b53178ed5641 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 22:01:41 +0000
Subject: [PATCH 43/49] Force shebang

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index ba2c6519c31c..1142e7b2716a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -385,7 +385,7 @@ def doTagAndPush(beatName, variant, sourceTag, targetTag) {
   def iterations = 0
   retryWithSleep(retries: 3, seconds: 5, backoff: true) {
     iterations++
-    def status = sh(label: "Change tag and push ${targetName}", script: """
+    def status = sh(label: "Change tag and push ${targetName}", script: """#!/usr/bin/env bash
       if docker image inspect "${sourceName}" &> /dev/null ; then
         docker tag ${sourceName} ${targetName}
         docker push ${targetName}

From 6931b00a18cf3283855d06bdd831a6bcb5af2ace Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Tue, 9 Feb 2021 22:33:51 +0000
Subject: [PATCH 44/49] debugging purposes

---
 Jenkinsfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Jenkinsfile b/Jenkinsfile
index 1142e7b2716a..40997219a0c1 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -386,6 +386,7 @@ def doTagAndPush(beatName, variant, sourceTag, targetTag) {
   retryWithSleep(retries: 3, seconds: 5, backoff: true) {
     iterations++
     def status = sh(label: "Change tag and push ${targetName}", script: """#!/usr/bin/env bash
+      docker images
       if docker image inspect "${sourceName}" &> /dev/null ; then
         docker tag ${sourceName} ${targetName}
         docker push ${targetName}

From 5ac1c0173023f35a9e44f9a7d861aec212661071 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 10 Feb 2021 08:38:08 +0000
Subject: [PATCH 45/49] Enable Snapshot

---
 Jenkinsfile | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 40997219a0c1..f70160a24dff 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -19,6 +19,7 @@ pipeline {
     PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}"
     RUNBLD_DISABLE_NOTIFICATIONS = 'true'
     SLACK_CHANNEL = "#beats-build"
+    SNAPSHOT = 'true'
     TERRAFORM_VERSION = "0.12.24"
     XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
   }
@@ -349,10 +350,12 @@ def pushCIDockerImages(beatsFolder){
 */
 def tagAndPush(beatName){
   def libbetaVer = env.VERSION
-  def aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
+  if("${env?.SNAPSHOT.trim()}" == "true"){
+    aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
 
-  libbetaVer += "-SNAPSHOT"
-  aliasVersion += "-SNAPSHOT"
+    libbetaVer += "-SNAPSHOT"
+    aliasVersion += "-SNAPSHOT"
+  }
 
   def tagName = "${libbetaVer}"
   if (isPR()) {

From 5817b85c511fb4fce7b46d9016f3cd262e0d4625 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 10 Feb 2021 14:20:19 +0000
Subject: [PATCH 46/49] Revert "Disable other pipelines"

This reverts commit 5c9d7ea36ed636efe7974142356b3f1957fe59cf.
---
 Jenkinsfile.yml | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
index 587bda77737b..284000da86fe 100644
--- a/Jenkinsfile.yml
+++ b/Jenkinsfile.yml
@@ -1,25 +1,25 @@
 projects:
-    #- "auditbeat"
-    #- "deploy/kubernetes"
-    #- "filebeat"
-    #- "generator"
-    #- "heartbeat"
-    #- "journalbeat"
-    #- "libbeat"
+    - "auditbeat"
+    - "deploy/kubernetes"
+    - "filebeat"
+    - "generator"
+    - "heartbeat"
+    - "journalbeat"
+    - "libbeat"
     - "metricbeat"
-    #- "packetbeat"
-    #- "winlogbeat"
-    #- "x-pack/auditbeat"
-    #- "x-pack/dockerlogbeat"
-    #- "x-pack/elastic-agent"
-    #- "x-pack/filebeat"
-    #- "x-pack/functionbeat"
-    #- "x-pack/heartbeat"
-    #- "x-pack/libbeat"
+    - "packetbeat"
+    - "winlogbeat"
+    - "x-pack/auditbeat"
+    - "x-pack/dockerlogbeat"
+    - "x-pack/elastic-agent"
+    - "x-pack/filebeat"
+    - "x-pack/functionbeat"
+    - "x-pack/heartbeat"
+    - "x-pack/libbeat"
     - "x-pack/metricbeat"
-    #- "x-pack/packetbeat"
-    #- "x-pack/winlogbeat"
-    #- "dev-tools"
+    - "x-pack/packetbeat"
+    - "x-pack/winlogbeat"
+    - "dev-tools"
     ##- "x-pack/journalbeat"  It is not yet in the 1.0 pipeline.
 
 ## Changeset macros that are defined here and used in each specific 2.0 pipeline.

From 7cad28aa8e144a33ae8dcc9363baa6bdfd841abf Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 10 Feb 2021 14:23:20 +0000
Subject: [PATCH 47/49] Restore previous configuration

---
 Jenkinsfile | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f70160a24dff..10ae8c528298 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -915,21 +915,21 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   }
   public run(Map args = [:]){
     def withModule = args.content.get('withModule', false)
-    /*if(args?.content?.containsKey('make')) {
+    if(args?.content?.containsKey('make')) {
       steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
     }
     if(args?.content?.containsKey('mage')) {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
-    }*/
+    }
     if(args?.content?.containsKey('packaging-linux')) {
       steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'), package: true)
-    }/*
+    }
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
     }
     if(args?.content?.containsKey('cloud')) {
       steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
-    } */
+    }
   }
 }
 

From 9849464507ec3f9ec10560e03c1acc99608d9783 Mon Sep 17 00:00:00 2001
From: Victor Martinez <VictorMartinezRubio@gmail.com>
Date: Wed, 10 Feb 2021 14:24:23 +0000
Subject: [PATCH 48/49] Disable e2e for the time being

---
 x-pack/metricbeat/Jenkinsfile.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
index df2e0b89b2aa..08aaa1d6fc0f 100644
--- a/x-pack/metricbeat/Jenkinsfile.yml
+++ b/x-pack/metricbeat/Jenkinsfile.yml
@@ -82,5 +82,5 @@ stages:
     packaging-linux:
         packaging-linux: "mage package"
         e2e:
-            enabled: true
+            enabled: false
             entrypoint: 'metricbeat-test.sh'

From 024b732844d40bdb2bf806480af2b03fcb8fbdbe Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Wed, 10 Feb 2021 18:10:26 +0000
Subject: [PATCH 49/49] Apply suggestions from code review

---
 Jenkinsfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 10ae8c528298..0aa41a1d5f8a 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -75,7 +75,7 @@ pipeline {
           withBeatsEnv(archive: false, id: "lint") {
             dumpVariables()
             setEnvVar('VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
-            /*whenTrue(env.ONLY_DOCS == 'true') {
+            whenTrue(env.ONLY_DOCS == 'true') {
               cmd(label: "make check", script: "make check")
             }
             whenTrue(env.ONLY_DOCS == 'false') {
@@ -83,7 +83,7 @@ pipeline {
               cmd(label: "make check-go", script: "make check-go")
               cmd(label: "make notice", script: "make notice")
               cmd(label: "Check for changes", script: "make check-no-changes")
-            }*/
+            }
           }
         }
       }