[CI] Enable packaging with Pipeline 2.0

Jenkinsfile

@@ -259,6 +259,61 @@ def k8sTest(Map args = [:]) {
   }
 }
 
+/**
+* This method runs the packaging
+*/
+def packagingLinux(Map args = [:]) {
+  def PLATFORMS = [ '+all',
+                'linux/amd64',
+                'linux/386',
+                'linux/arm64',
+                'linux/armv7',
+                // The platforms above are disabled temporarly as crossbuild images are
+                // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
+                //'linux/ppc64le',
+                //'linux/mips64',
+                //'linux/s390x',
+                'windows/amd64',
+                'windows/386',
+                (params.macos ? '' : 'darwin/amd64'),
+              ].join(' ')
+  withEnv([
+    "PLATFORMS=${PLATFORMS}"
+  ]) {
+    target(args)
+  }
+}
+
+def e2e(Map args = [:]) {
+  def suite = args.get('suite', '')
+  def tags = args.get('tags', '')
+  def stackVersion = args.get('stackVersion', '8.0.0-SNAPSHOT')  // TBC with the version defined somewhere...
+  dir('.e2e') {
+    gitCheckout(
+      branch: "master",  // TBC with the target branch if running on a PR basis.
+      repo: "https://github.com/elastic/e2e-testing.git",
+      credentialsId: "${JOB_GIT_CREDENTIALS}"
+    )
+    try {
+      if(isInstalled(tool: 'docker', flag: '--version')) {
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+      }
+      retry(3){
+        sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
+      }
+      filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
+        // TBC with the suite to be used
+        sh script: """SUITE=metricbeat DEVELOPER_MODE=false TIMEOUT_FACTOR=3 LOG_LEVEL=TRACE make -C e2e functional-test""", label: "Run functional tests for ${suite}:${tags}"
+      }
+    } catch(e) {
+      error(e.toString())
+    } finally {
+      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
+        archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+    }
+  }
+}
+
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false
@@ -270,6 +325,7 @@ def target(Map args = [:]) {
   def directory = args.get('directory', '')
   def withModule = args.get('withModule', false)
   def isMage = args.get('isMage', false)
+  def isE2E = args.get('isE2E', false)
   withNode(args.label) {
     withGithubNotify(context: "${context}") {
       withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
@@ -279,6 +335,9 @@ def target(Map args = [:]) {
         dir(isMage ? directory : '') {
           cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
         }
+        if(isE2E) {
+          e2e(suite: directory)
+        }
       }
     }
   }
@@ -708,18 +767,22 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   }
   public run(Map args = [:]){
     def withModule = args.content.get('withModule', false)
-    if(args?.content?.containsKey('make')) {
+    def isE2E = args.content.get('isE2E', false)
+    /*if(args?.content?.containsKey('make')) {
       steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
     }
     if(args?.content?.containsKey('mage')) {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
-    }
+    }*/
+    if(args?.content?.containsKey('packaging-linux')) {
+      steps.packagingLinux(context: args.context, command: args.content.packaging-linux, directory: args.project, label: args.label, isMage: true, id: args.id, isE2E: isE2E)
+    }/*
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
     }
     if(args?.content?.containsKey('cloud')) {
       steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
-    }
+    } */
   }
 }
 

Jenkinsfile.yml

@@ -14,13 +14,13 @@ projects:
     - "x-pack/elastic-agent"
     - "x-pack/filebeat"
     - "x-pack/functionbeat"
+    - "x-pack/heartbeat"
     - "x-pack/libbeat"
     - "x-pack/metricbeat"
     - "x-pack/packetbeat"
     - "x-pack/winlogbeat"
     - "dev-tools"
-    ##- "x-pack/heartbeat"    It's not yet in the 1.0 pipeline.
-    ##- "x-pack/journalbeat"  It's not yet in the 1.0 pipeline.
+    ##- "x-pack/journalbeat"  It is not yet in the 1.0 pipeline.
 
 ## Changeset macros that are defined here and used in each specific 2.0 pipeline.
 changeset:

auditbeat/Jenkinsfile.yml

@@ -83,3 +83,6 @@ stages:
     #            - "windows-7"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

filebeat/Jenkinsfile.yml

@@ -81,3 +81,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

heartbeat/Jenkinsfile.yml

@@ -80,3 +80,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

journalbeat/Jenkinsfile.yml

@@ -33,3 +33,6 @@ stages:
             tags: true         ## for all the tags
     unitTest:
         mage: "mage build unitTest"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

metricbeat/Jenkinsfile.yml

@@ -75,3 +75,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: true

packetbeat/Jenkinsfile.yml

@@ -81,3 +81,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

winlogbeat/Jenkinsfile.yml

@@ -51,3 +51,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/auditbeat/Jenkinsfile.yml

@@ -81,3 +81,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/dockerlogbeat/Jenkinsfile.yml

@@ -21,3 +21,6 @@ stages:
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/elastic-agent/Jenkinsfile.yml

@@ -85,3 +85,6 @@ stages:
     #            - "windows-7-32"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/filebeat/Jenkinsfile.yml

@@ -88,3 +88,6 @@ stages:
     #            - "windows-7-32"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/functionbeat/Jenkinsfile.yml

@@ -76,3 +76,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/heartbeat/Jenkinsfile.yml

@@ -0,0 +1,18 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^x-pack/heartbeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@xpack"             ## special token regarding the changeset for the xpack
+    comments:                  ## when PR comment contains any of those entries
+        - "/test x-pack/heartbeat"
+    labels:                    ## when PR labels matches any of those entries
+        - "x-pack-heartbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "x-pack-heartbeat"
+    tags: true                 ## for all the tags
+platform: "immutable && ubuntu-18" ## default label for all the stages
+stages:
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/metricbeat/Jenkinsfile.yml

@@ -79,3 +79,6 @@ stages:
 #        mage: "mage build unitTest"
 #        platforms:             ## override default labels in this specific stage.
 #            - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/packetbeat/Jenkinsfile.yml

@@ -82,3 +82,6 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false

x-pack/winlogbeat/Jenkinsfile.yml

@@ -55,3 +55,8 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        isE2E: false
+        platforms:             ## override default labels in this specific stage.
+            - "immutable && ubuntu-18"