Cherry-pick to 7.10: packaging backports (elastic#22076)

* feat: add a new step to run the e2e tests for certain parts of Beats (elastic#21100)

* feat: add a new step to run the e2e tests for certain parts of Beats

We are going to trigger the tests for those parts affected by the
elastic-agent, filebeat, or metricbeat, because those are the ones we
verify  in the e2e-testing suite

* chore: do not include heartbeat

* feat: trigger the e2e tests

* fix: use relative path

* chore: use proper target branch name for PRs

* chore: use different tag

* fix: use proper env variable

* chore: pass github checks context to downstream job

* chore: revert shared lib version

Co-authored-by: Victor Martinez <[email protected]>

* chore: add BASE_DIR env variable

Co-authored-by: Victor Martinez <[email protected]>

* chore: remove duplicated env

* ffix: add param comma separator

* fix: wrong copy&paste

* chore: move e2e GH check out of the release context

* chore: simplify conditional logic

* chore: refine execution of test suites

* fix: use proper parameter name

* chore: set metricbeat version

* chore: remove slack notifications on PRs

* chore: update parameter

* chore: run multiple test suites per beat type

Co-authored-by: Victor Martinez <[email protected]>

* [E2E Tests] fix: set versions ony for PRs (elastic#21608)

* fix: set versions ony for PRs

We want to use default versions per branch when running after a merge

* fix: add trailing comma

Co-authored-by: Victor Martinez <[email protected]>

Co-authored-by: Victor Martinez <[email protected]>

* [CI: Packaging] fix: push ubi8 images too (elastic#21621)

* fix: push ubi8 images too

* chore: enhance retries

Co-authored-by: Victor Martinez <[email protected]>

* chore: use variables in log

* chore: add "-oss" images

Co-authored-by: Victor Martinez <[email protected]>

* fix: remove extra curly brace in script (elastic#21692)

* fix: remove extra curly brace

* chore: proper indent

* fix: update fleet test suite name (elastic#21738)

* chore: create CI artifacts for DEV usage (elastic#21645)

It will create the artifacts with some requirements related to integrity

* chore: simplify triggering the E2E tests for Beats (elastic#21790)

* chore: pass beat as a method argument (no side-effects)

* chore: run tests in a separate stage

* fix: use parenthesis

* chore: update comment

* chore: do not trigger E2E tests if no suite was added

* fix: use missing curly brackets

* fix: wrong closure wrapping

* fix: condition was not set

* chore: delegate variant pushes to the right method (elastic#21861)

* fix: delegate pushes to variants

* chore: group conditions for x-pack

* chore: simplify with endsWith

Co-authored-by: Victor Martinez <[email protected]>

Co-authored-by: Victor Martinez <[email protected]>

* feat: package aliases for snapshots (elastic#21960)

* feat: push aliases for docker images

* feat: build alias for snapshots

* fix: only update alias on snapshots

Co-authored-by: Jaime Soriano Pastor <[email protected]>

* fix: wrong image name for alias

* fix: reuse variable as groovy does not hide variables by scope

* chore: extract common logic to a method

* Revert "fix: only update alias on snapshots"

This reverts commit cff2cef.

* Revert "feat: build alias for snapshots"

This reverts commit 707e0d7.

* chore: do not push aliases for PRs

Co-authored-by: Jaime Soriano Pastor <[email protected]>

* chore: Use third number as x

Co-authored-by: Victor Martinez <[email protected]>
Co-authored-by: Jaime Soriano Pastor <[email protected]>
(cherry picked from commit 624c459)

.ci/packaging.groovy

@@ -2,15 +2,24 @@
 
 @Library('apm@current') _
 
+import groovy.transform.Field
+
+/**
+ This is required to store the test suites we will use to trigger the E2E tests.
+*/
+@Field def e2eTestSuites = []
+
 pipeline {
   agent none
   environment {
-    BASE_DIR = 'src/github.com/elastic/beats'
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
+    GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
     SNAPSHOT = "true"
     PIPELINE_LOG_LEVEL = "INFO"
   }
@@ -34,7 +43,7 @@ pipeline {
   }
   stages {
     stage('Filter build') {
-      agent { label 'ubuntu && immutable' }
+      agent { label 'ubuntu-18 && immutable' }
       when {
         beforeAgent true
         anyOf {
@@ -103,14 +112,14 @@ pipeline {
                    'x-pack/heartbeat',
                   // 'x-pack/journalbeat',
                   'x-pack/metricbeat',
-                  'x-pack/packetbeat',
+                  // 'x-pack/packetbeat',
                   'x-pack/winlogbeat'
                 )
               }
             }
             stages {
               stage('Package Linux'){
-                agent { label 'ubuntu && immutable' }
+                agent { label 'ubuntu-18 && immutable' }
                 options { skipDefaultCheckout() }
                 when {
                   beforeAgent true
@@ -142,6 +151,7 @@ pipeline {
                     release()
                     pushCIDockerImages()
                   }
+                  prepareE2ETestForPackage("${BEATS_FOLDER}")
                 }
               }
               stage('Package Mac OS'){
@@ -172,6 +182,13 @@ pipeline {
             }
           }
         }
+        stage('Run E2E Tests for Packages'){
+          agent { label 'ubuntu-18 && immutable' }
+          options { skipDefaultCheckout() }
+          steps {
+            runE2ETests()
+          }
+        }
       }
       post {
         success {
@@ -190,68 +207,160 @@ pipeline {
 
 def pushCIDockerImages(){
   catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
-    if ("${env.BEATS_FOLDER}" == "auditbeat"){
-      tagAndPush('auditbeat-oss')
-    } else if ("${env.BEATS_FOLDER}" == "filebeat") {
-      tagAndPush('filebeat-oss')
-    } else if ("${env.BEATS_FOLDER}" == "heartbeat"){
-      tagAndPush('heartbeat-oss')
+    if (env?.BEATS_FOLDER?.endsWith('auditbeat')) {
+      tagAndPush('auditbeat')
+    } else if (env?.BEATS_FOLDER?.endsWith('filebeat')) {
+      tagAndPush('filebeat')
+    } else if (env?.BEATS_FOLDER?.endsWith('heartbeat')) {
+      tagAndPush('heartbeat')
     } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
       tagAndPush('journalbeat')
-      tagAndPush('journalbeat-oss')
-    } else if ("${env.BEATS_FOLDER}" == "metricbeat"){
-      tagAndPush('metricbeat-oss')
+    } else if (env?.BEATS_FOLDER?.endsWith('metricbeat')) {
+      tagAndPush('metricbeat')
     } else if ("${env.BEATS_FOLDER}" == "packetbeat"){
       tagAndPush('packetbeat')
-      tagAndPush('packetbeat-oss')
-    } else if ("${env.BEATS_FOLDER}" == "x-pack/auditbeat"){
-      tagAndPush('auditbeat')
     } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
       tagAndPush('elastic-agent')
-    } else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
-      tagAndPush('filebeat')
-    } else if ("${env.BEATS_FOLDER}" == "x-pack/heartbeat"){
-        tagAndPush('heartbeat')
-    } else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
-      tagAndPush('metricbeat')
     }
   }
 }
 
-def tagAndPush(name){
+def tagAndPush(beatName){
   def libbetaVer = env.BEAT_VERSION
+  def aliasVersion = ""
   if("${env.SNAPSHOT}" == "true"){
+    aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
+
     libbetaVer += "-SNAPSHOT"
+    aliasVersion += "-SNAPSHOT"
   }
 
   def tagName = "${libbetaVer}"
   if (isPR()) {
     tagName = "pr-${env.CHANGE_ID}"
   }
 
-  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
-  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}"
-  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
   dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
-  retry(3){
-    sh(label:'Change tag and push', script: """
-      docker tag ${oldName} ${newName}
-      docker push ${newName}
-      docker tag ${oldName} ${commitName}
-      docker push ${commitName}
-    """)
+
+  // supported image flavours
+  def variants = ["", "-oss", "-ubi8"]
+  variants.each { variant ->
+    doTagAndPush(beatName, variant, libbetaVer, tagName)
+    doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
+
+    if (!isPR() && aliasVersion != "") {
+      doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+    }
+  }
+}
+
+/**
+* @param beatName name of the Beat
+* @param variant name of the variant used to build the docker image name
+* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
+* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
+*/
+def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+  def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+  def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
+
+  def iterations = 0
+  retryWithSleep(retries: 3, seconds: 5, backoff: true) {
+    iterations++
+    def status = sh(label: "Change tag and push ${targetName}", script: """
+      docker tag ${sourceName} ${targetName}
+      docker push ${targetName}
+    """, returnStatus: true)
+
+    if ( status > 0 && iterations < 3) {
+      error("tag and push failed for ${beatName}, retry")
+    } else if ( status > 0 ) {
+      log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
+    }
+  }
+}
+
+def prepareE2ETestForPackage(String beat){
+  if ("${beat}" == "filebeat" || "${beat}" == "x-pack/filebeat") {
+    e2eTestSuites.push('fleet')
+    e2eTestSuites.push('helm')
+  } else if ("${beat}" == "metricbeat" || "${beat}" == "x-pack/metricbeat") {
+    e2eTestSuites.push('ALL')
+    echo("${beat} adds all test suites to the E2E tests job.")
+  } else if ("${beat}" == "x-pack/elastic-agent") {
+    e2eTestSuites.push('fleet')
+  } else {
+    echo("${beat} does not add any test suite to the E2E tests job.")
+    return
   }
 }
 
 def release(){
   withBeatsEnv(){
-    dir("${env.BEATS_FOLDER}") {
-      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    withEnv([
+      "DEV=true"
+    ]) {
+      dir("${env.BEATS_FOLDER}") {
+        sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+      }
     }
     publishPackages("${env.BEATS_FOLDER}")
   }
 }
 
+def runE2ETests(){
+  if (e2eTestSuites.size() == 0) {
+    echo("Not triggering E2E tests for PR-${env.CHANGE_ID} because the changes does not affect the E2E.")
+    return
+  }
+
+  def suites = '' // empty value represents all suites in the E2E tests
+
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
+    def suitesSet = e2eTestSuites.toSet()
+
+    if (!suitesSet.contains('ALL')) {
+      suitesSet.each { suite ->
+        suites += "${suite},"
+      };
+    }
+
+    triggerE2ETests(suites)
+  }
+}
+
+def triggerE2ETests(String suite) {
+  echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")
+
+  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}.x"
+  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+
+  def parameters = [
+    booleanParam(name: 'forceSkipGitChecks', value: true),
+    booleanParam(name: 'forceSkipPresubmit', value: true),
+    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    string(name: 'runTestsSuites', value: suite),
+    string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
+    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
+  ]
+  if (isPR()) {
+    def version = "pr-${env.CHANGE_ID}"
+    parameters.push(booleanParam(name: 'USE_CI_SNAPSHOTS', value: true))
+    parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
+    parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
+  }
+
+  build(job: "${e2eTestsPipeline}",
+    parameters: parameters,
+    propagate: false,
+    wait: false
+  )
+
+  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME}"
+  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
+}
+
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],