Backport some features in Jenkinsfile to 7.x

Jenkinsfile

@@ -8,6 +8,9 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/beats'
     GOX_FLAGS = "-arch amd64"
     DOCKER_COMPOSE_VERSION = "1.21.0"
+    PIPELINE_LOG_LEVEL = "INFO"
+    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
   }
   options {
     timeout(time: 2, unit: 'HOURS')
@@ -17,7 +20,6 @@ pipeline {
     disableResume()
     durabilityHint('PERFORMANCE_OPTIMIZED')
     disableConcurrentBuilds()
-//    checkoutToSubdirectory "${env.BASE_DIR}"
   }
   triggers {
     issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
@@ -27,6 +29,7 @@ pipeline {
     booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
     booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
     booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
+    booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
   }
   stages {
     /**
@@ -37,13 +40,13 @@ pipeline {
       steps {
         deleteDir()
         gitCheckout(basedir: "${BASE_DIR}")
+        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
         dir("${BASE_DIR}"){
           loadConfigEnvVars()
         }
         whenTrue(params.debug){
           dumpFilteredEnvironment()
         }
-        stash allowEmpty: true, name: 'source', useDefaultExcludes: false
       }
     }
     stage('Lint'){
@@ -644,12 +647,20 @@ def withBeatsEnv(boolean archive, Closure body) {
   ]) {
     deleteDir()
     unstash 'source'
+    if(isDockerInstalled()){
+      dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+    }
     dir("${env.BASE_DIR}") {
       sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh")
       sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh")
       sh(label: "Install Mage", script: "make mage")
+      // TODO (2020-04-07): This is a work-around to fix the Beat generator tests.
+      // See https://github.com/elastic/beats/issues/17787.
+      setGitConfig()
       try {
-        body()
+        if(!params.dry_run){
+          body()
+        }
       } finally {
         if (archive) {
           catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
@@ -682,7 +693,9 @@ def withBeatsEnvWin(Closure body) {
     dir("${env.BASE_DIR}"){
       bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat")
       try {
-        body()
+        if(!params.dry_run){
+          body()
+        }
       } finally {
         catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
           junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml")
@@ -818,19 +831,20 @@ def isChanged(patterns){
 }
 
 def isChangedOSSCode(patterns) {
-  def always = [
+  def allPatterns = [
     "^Jenkinsfile",
     "^vendor/.*",
     "^libbeat/.*",
     "^testing/.*",
     "^dev-tools/.*",
     "^\\.ci/.*",
   ]
-  return isChanged(always + patterns)
+  allPatterns.addAll(patterns)
+  return isChanged(allPatterns)
 }
 
 def isChangedXPackCode(patterns) {
-  def always = [
+  def allPatterns = [
     "^Jenkinsfile",
     "^vendor/.*",
     "^libbeat/.*",
@@ -839,91 +853,112 @@ def isChangedXPackCode(patterns) {
     "^x-pack/libbeat/.*",
     "^\\.ci/.*",
   ]
-  return isChanged(always + patterns)
+  allPatterns.addAll(patterns)
+  return isChanged(allPatterns)
 }
 
 def loadConfigEnvVars(){
+  def empty = []
+  env.GO_VERSION = readFile(".go-version").trim()
+
+  withEnv(["HOME=${env.WORKSPACE}"]) {
+    sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh")
+  }
 
   // Libbeat is the core framework of Beats. It has no additional dependencies
   // on other projects in the Beats repository.
-  env.BUILD_LIBBEAT = isChangedOSSCode([])
-  env.BUILD_LIBBEAT_XPACK = env.BUILD_LIBBEAT || isChangedXPackCode([])
+  env.BUILD_LIBBEAT = isChangedOSSCode(empty)
+  env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty)
 
   // Auditbeat depends on metricbeat as framework, but does not include any of
   // the modules from Metricbeat.
   // The Auditbeat x-pack build contains all functionality from OSS Auditbeat.
-  env.BUILD_AUDITBEAT = isChangedOSSCode([
-    "^metricbeat/.*",
-    "^auditbeat/.*",
-  ])
-  env.BUILD_AUDITBEAT_XPACK = env.BUILD_AUDITBEAT || isChangedXPackCode([
-    "^x-pack/auditbeat/.*",
-  ])
+  env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat'))
+  env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat'))
 
   // Dockerlogbeat is a standalone Beat that only relies on libbeat.
-  env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode([
-    "^x-pack/dockerlogbeat/.*",
-  ])
+  env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat'))
 
   // Filebeat depends on libbeat only.
   // The Filebeat x-pack build contains all functionality from OSS Filebeat.
-  env.BUILD_FILEBEAT = isChangedOSSCode(["^filebeat/.*"])
-  env.BUILD_FILEBEAT_XPACK = env.BUILD_FILEBEAT || isChangedXPackCode([
-    "^x-pack/filebeat/.*",
-  ])
+  env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat'))
+  env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat'))
 
   // Metricbeat depends on libbeat only.
   // The Metricbeat x-pack build contains all functionality from OSS Metricbeat.
-  env.BUILD_METRICBEAT = isChangedOSSCode(["^metricbeat/.*"])
-  env.BUILD_METRICBEAT_XPACK = env.BUILD_METRICBEAT || isChangedXPackCode([
-    "^x-pack/metricbeat/.*",
-  ])
+  env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat'))
+  env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat'))
 
   // Functionbeat is a standalone beat that depends on libbeat only.
   // Functionbeat is available as x-pack build only.
-  env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode([
-    "^x-pack/functionbeat/.*",
-  ])
+  env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat'))
 
   // Heartbeat depends on libbeat only.
   // The Heartbeat x-pack build contains all functionality from OSS Heartbeat.
-  env.BUILD_HEARTBEAT = isChangedOSSCode(["^heartbeat/.*"])
-  env.BUILD_HEARTBEAT_XPACK = env.BUILD_HEARTBEAT || isChangedXPackCode([
-    "^x-pack/heartbeat/.*",
-  ])
-
+  env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat'))
+  env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat'))
 
   // Journalbeat depends on libbeat only.
   // The Journalbeat x-pack build contains all functionality from OSS Journalbeat.
-  env.BUILD_JOURNALBEAT = isChangedOSSCode(["^journalbeat/.*"])
-  env.BUILD_JOURNALBEAT_XPACK = env.BUILD_JOURNALBEAT || isChangedXPackCode([
-    "^x-pack/journalbeat/.*",
-  ])
+  env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat'))
+  env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat'))
 
   // Packetbeat depends on libbeat only.
   // The Packetbeat x-pack build contains all functionality from OSS Packetbeat.
-  env.BUILD_PACKETBEAT = isChangedOSSCode(["^packetbeat/.*"])
-  env.BUILD_PACKETBEAT_XPACK = env.BUILD_PACKETBEAT || isChangedXPackCode([
-    "^x-pack/packetbeat/.*",
-  ])
+  env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat'))
+  env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat'))
 
   // Winlogbeat depends on libbeat only.
   // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat.
-  env.BUILD_WINLOGBEAT = isChangedOSSCode(["^winlogbeat/.*"])
-  env.BUILD_WINLOGBEAT_XPACK = env.BUILD_WINLOGBEAT || isChangedXPackCode([
-    "^x-pack/winlogbeat/.*",
-  ])
+  env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat'))
+  env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat'))
+
+  // Elastic-agent is a self-contained product, that depends on libbeat only.
+  // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat.
+  // The agent is available as x-pack build only.
+  env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/elastic-agent'))
 
   // The Kubernetes test use Filebeat and Metricbeat, but only need to be run
   // if the deployment scripts have been updated. No Beats specific testing is
   // involved.
   env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"])
 
-  env.BUILD_GENERATOR = isChangedOSSCode(["^generator/.*"])
+  def generatorPatterns = ['^generator/.*']
+  generatorPatterns.addAll(getVendorPatterns('generator/common/beatgen'))
+  generatorPatterns.addAll(getVendorPatterns('metricbeat/beater'))
+  env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns)
+}
+
+/**
+  This method grab the dependencies of a Go module and transform them on regexp
+*/
+def getVendorPatterns(beatName){
+  def os = goos()
+  def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64"
+  def output = ""
 
-  env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode([
-    "^x-pack/elastic-agent/.*",
-  ])
+  withEnv([
+    "HOME=${env.WORKSPACE}/${env.BASE_DIR}",
+    "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}",
+  ]) {
+    output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """
+      go list -mod=vendor -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\
+        |awk '{print \$1"/.*"}'\
+        |sed -e "s#github.com/elastic/beats/v7/##g"
+    """)
+  }
+  return output?.split('\n').collect{ item -> item as String }
+}
 
-  env.GO_VERSION = readFile(".go-version").trim()
+def setGitConfig(){
+  sh(label: 'check git config', script: '''
+    if [ -z "$(git config --get user.email)" ]; then
+      git config user.email "[email protected]"
+      git config user.name "beatsmachine"
+    fi
+  ''')
+}
+
+def isDockerInstalled(){
+  return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true)
 }

x-pack/metricbeat/docker-compose.yml

@@ -10,6 +10,7 @@ services:
     volumes:
       - ${PWD}/../..:/go/src/github.com/elastic/beats/
       - /var/run/docker.sock:/var/run/docker.sock
+      - ${HOME}/.docker:/root/.docker:ro
     network_mode: host
     command: make