[CI] Enable packaging with Pipeline 2.0

Jenkinsfile

@@ -9,7 +9,7 @@ pipeline {
     AWS_REGION = "${params.awsRegion}"
     REPO = 'beats'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
-    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_COMPOSE_VERSION = "1.21.0"
     DOCKER_REGISTRY = 'docker.elastic.co'
     JOB_GCS_BUCKET = 'beats-ci-temp'
@@ -66,6 +66,9 @@ pipeline {
     }
     stage('Lint'){
       options { skipDefaultCheckout() }
+      when {
+        expression { return false }
+      }
       environment {
         GOFLAGS = '-mod=readonly'
       }
@@ -259,17 +262,203 @@ def k8sTest(Map args = [:]) {
   }
 }
 
+/**
+* This method runs the packaging
+*/
+def packagingLinux(Map args = [:]) {
+  def PLATFORMS = [ '+all',
+                'linux/amd64',
+                'linux/386',
+                'linux/arm64',
+                'linux/armv7',
+                // The platforms above are disabled temporarly as crossbuild images are
+                // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
+                //'linux/ppc64le',
+                //'linux/mips64',
+                //'linux/s390x',
+                'windows/amd64',
+                'windows/386',
+                (params.macos ? '' : 'darwin/amd64'),
+              ].join(' ')
+  withEnv([
+    "PLATFORMS=${PLATFORMS}"
+  ]) {
+    target(args)
+  }
+}
+
+
+/**
+* Upload the packages to their snapshot or pull request buckets
+* @param beatsFolder beats folder
+*/
+def publishPackages(beatsFolder){
+  def bucketUri = "gs://beats-ci-artifacts/snapshots"
+  if (isPR()) {
+    bucketUri = "gs://beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}"
+  }
+  def beatsFolderName = getBeatsName(beatsFolder)
+  uploadPackages("${bucketUri}/${beatsFolderName}", beatsFolder)
+
+  // Copy those files to another location with the sha commit to test them
+  // afterward.
+  bucketUri = "gs://beats-ci-artifacts/commits/${env.GIT_BASE_COMMIT}"
+  uploadPackages("${bucketUri}/${beatsFolderName}", beatsFolder)
+}
+
+/**
+* Upload the distribution files to google cloud.
+* TODO: There is a known issue with Google Storage plugin.
+* @param bucketUri the buckets URI.
+* @param beatsFolder the beats folder.
+*/
+def uploadPackages(bucketUri, beatsFolder){
+  googleStorageUpload(bucket: bucketUri,
+    credentialsId: "${JOB_GCS_CREDENTIALS}",
+    pathPrefix: "${beatsFolder}/build/distributions/",
+    pattern: "${beatsFolder}/build/distributions/**/*",
+    sharedPublicly: true,
+    showInline: true
+  )
+}
+
+/**
+* Push the docker images for the given beat.
+* @param beatsFolder beats folder
+*/
+def pushCIDockerImages(beatsFolder){
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    if (beatsFolder.endsWith('auditbeat')) {
+      tagAndPush('auditbeat')
+    } else if (beatsFolder.endsWith('filebeat')) {
+      tagAndPush('filebeat')
+    } else if (beatsFolder.endsWith('heartbeat')) {
+      tagAndPush('heartbeat')
+    } else if ("${beatsFolder}" == "journalbeat"){
+      tagAndPush('journalbeat')
+    } else if (beatsFolder.endsWith('metricbeat')) {
+      tagAndPush('metricbeat')
+    } else if ("${beatsFolder}" == "packetbeat"){
+      tagAndPush('packetbeat')
+    } else if ("${beatsFolder}" == "x-pack/elastic-agent") {
+      tagAndPush('elastic-agent')
+    }
+  }
+}
+
+/**
+* Tag and push all the docker images for the given beat.
+* @param beatName name of the Beat
+*/
+def tagAndPush(beatName){
+  def libbetaVer = env.VERSION
+  def aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
+
+  libbetaVer += "-SNAPSHOT"
+  aliasVersion += "-SNAPSHOT"
+
+  def tagName = "${libbetaVer}"
+  if (isPR()) {
+    tagName = "pr-${env.CHANGE_ID}"
+  }
+
+  // supported image flavours
+  def variants = ["", "-oss", "-ubi8"]
+  variants.each { variant ->
+    doTagAndPush(beatName, variant, libbetaVer, tagName)
+    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
+
+    if (!isPR() && aliasVersion != "") {
+      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    }
+  }
+}
+
+/**
+* Tag and push the given sourceTag docker image with the tag name targetTag.
+* @param beatName name of the Beat
+* @param variant name of the variant used to build the docker image name
+* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
+* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
+*/
+def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
+
+  def iterations = 0
+  retryWithSleep(retries: 3, seconds: 5, backoff: true) {
+    iterations++
+    def status = sh(label: "Change tag and push ${targetName}", script: """
+      if docker image inspect "${sourceName}" &> /dev/null ; then
+        docker tag ${sourceName} ${targetName}
+        docker push ${targetName}
+      else
+        echo 'docker image ${sourceName} does not exist'
+      fi
+    """, returnStatus: true)
+    if ( status > 0 && iterations < 3) {
+      error("tag and push failed for ${beatName}, retry")
+    } else if ( status > 0 ) {
+      log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
+    }
+  }
+}
+
+/**
+* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
+* therefore the storage bucket in GCP should follow the same folder structure.
+* This is required by https://github.com/elastic/beats-tester
+* e.g.
+* baseDir=name -> return name
+* baseDir=name1/name2/name3-> return name2
+*/
+def getBeatsName(baseDir) {
+  return baseDir.replace('x-pack/', '')
+}
+
+/**
+* This method runs the end 2 end testing in the same worker where the packages have been
+* generated, this should help to speed up the things
+*/
+def e2e(Map args = [:]) {
+  def enabled = args.e2e?.get('enabled', false)
+  def entrypoint = args.e2e?.get('entrypoint')
+  if (!enabled) { return }
+  dir("${env.WORKSPACE}/src/github.com/elastic/e2e-testing") {
+    // TBC with the target branch if running on a PR basis.
+    git(branch: 'master', credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken', url: 'https://github.com/elastic/e2e-testing.git')
+    try {
+      if(isDockerInstalled()) {
+        dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+      }
+      def goVersionForE2E = readFile('.go-version').trim()
+      withEnv(["GO_VERSION=${goVersionForE2E}",
+               "BEATS_LOCAL_PATH=${env.WORKSPACE}/${env.BASE_DIR}"]) {
+        filebeat(output: "docker_logs_${entrypoint}.log", workdir: "${env.WORKSPACE}"){
+          sh script: ".ci/scripts/${entrypoint}", label: "Run functional tests ${entrypoint}"
+        }
+      }
+    } catch(e) {
+      error(e.toString())
+    } finally {
+      junit(allowEmptyResults: true, keepLongStdio: true, testResults: "outputs/TEST-*.xml")
+      archiveArtifacts allowEmptyArchive: true, artifacts: "outputs/TEST-*.xml"
+    }
+  }
+}
+
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false
 *  - mage then the dir(location) is required, aka by enabling isMage: true.
 */
 def target(Map args = [:]) {
-  def context = args.context
   def command = args.command
+  def context = args.context
   def directory = args.get('directory', '')
   def withModule = args.get('withModule', false)
   def isMage = args.get('isMage', false)
+  def isE2E = args.e2e?.get('enabled', false)
   withNode(args.label) {
     withGithubNotify(context: "${context}") {
       withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
@@ -279,6 +468,11 @@ def target(Map args = [:]) {
         dir(isMage ? directory : '') {
           cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
         }
+        if(isE2E) {
+          e2e(args)
+        }
+        publishPackages("${directory}")
+        pushCIDockerImages("${directory}")
       }
     }
   }
@@ -354,7 +548,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
     "USERPROFILE=${userProfile}"
   ]) {
     if(isDockerInstalled()) {
-      dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+      dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
     }
     dir("${env.BASE_DIR}") {
       installTools(args)
@@ -675,7 +869,7 @@ def dumpVariables(){
 def isDockerInstalled(){
   if (isUnix()) {
     // TODO: some issues with macosx if(isInstalled(tool: 'docker', flag: '--version')) {
-    return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true)
+    return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) == 0
   } else {
     return false
   }
@@ -708,18 +902,21 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   }
   public run(Map args = [:]){
     def withModule = args.content.get('withModule', false)
-    if(args?.content?.containsKey('make')) {
+    /*if(args?.content?.containsKey('make')) {
       steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id)
     }
     if(args?.content?.containsKey('mage')) {
       steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
-    }
+    }*/
+    if(args?.content?.containsKey('packaging-linux')) {
+      steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), directory: args.project, label: args.label, isMage: true, id: args.id, e2e: args.content.get('e2e'))
+    }/*
     if(args?.content?.containsKey('k8sTest')) {
       steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
     }
     if(args?.content?.containsKey('cloud')) {
       steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
-    }
+    } */
   }
 }
 

Jenkinsfile.yml

@@ -14,13 +14,13 @@ projects:
     - "x-pack/elastic-agent"
     - "x-pack/filebeat"
     - "x-pack/functionbeat"
+    - "x-pack/heartbeat"
     - "x-pack/libbeat"
     - "x-pack/metricbeat"
     - "x-pack/packetbeat"
     - "x-pack/winlogbeat"
     - "dev-tools"
-    ##- "x-pack/heartbeat"    It's not yet in the 1.0 pipeline.
-    ##- "x-pack/journalbeat"  It's not yet in the 1.0 pipeline.
+    ##- "x-pack/journalbeat"  It is not yet in the 1.0 pipeline.
 
 ## Changeset macros that are defined here and used in each specific 2.0 pipeline.
 changeset:

auditbeat/Jenkinsfile.yml

@@ -83,3 +83,7 @@ stages:
     #            - "windows-7"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

filebeat/Jenkinsfile.yml

@@ -81,3 +81,7 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

heartbeat/Jenkinsfile.yml

@@ -80,3 +80,7 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

journalbeat/Jenkinsfile.yml

@@ -33,3 +33,7 @@ stages:
             tags: true         ## for all the tags
     unitTest:
         mage: "mage build unitTest"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

metricbeat/Jenkinsfile.yml

@@ -75,3 +75,8 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: true
+            entrypoint: 'metricbeat-test.sh'

packetbeat/Jenkinsfile.yml

@@ -81,3 +81,7 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

winlogbeat/Jenkinsfile.yml

@@ -51,3 +51,7 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

x-pack/auditbeat/Jenkinsfile.yml

@@ -81,3 +81,7 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

x-pack/dockerlogbeat/Jenkinsfile.yml

@@ -21,3 +21,7 @@ stages:
     build:
         mage: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

x-pack/elastic-agent/Jenkinsfile.yml

@@ -85,3 +85,7 @@ stages:
     #            - "windows-7-32"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

x-pack/filebeat/Jenkinsfile.yml

@@ -88,3 +88,7 @@ stages:
     #            - "windows-7-32"
     #        branches: true     ## for all the branches
     #        tags: true         ## for all the tags
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false

x-pack/functionbeat/Jenkinsfile.yml

@@ -76,3 +76,7 @@ stages:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
             - "windows-7-32-bit"
+    packaging-linux:
+        packaging-linux: "mage package"
+        e2e:
+            enabled: false