Jenkinsfile

#!/usr/bin/env groovy

@Library('apm@current') _

pipeline {
  agent any
  environment {
    REPO = 'e2e-testing'
    BASE_DIR = "src/github.com/elastic/${env.REPO}"
    ELASTIC_APM_ACTIVE="true"
    ELASTIC_APM_ENVIRONMENT="ci"
    ELASTIC_APM_LOG_FILE="stderr"
    ELASTIC_APM_LOG_LEVEL="debug"
    VAULT_INSTRUMENTATION_SECRET="secret/observability-team/ci/jenkins-stats"
    NIGHTLY_TAG="@nightly"
    NOTIFY_TO = credentials('notify-to')
    JOB_GCS_BUCKET = credentials('gcs-bucket')
    JOB_GIT_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken"
    DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
    DOCKER_REGISTRY = 'docker.elastic.co'
  }
  options {
    timeout(time: 1, unit: 'HOURS')
    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
    timestamps()
    ansiColor('xterm')
    disableResume()
    durabilityHint('PERFORMANCE_OPTIMIZED')
    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
    quietPeriod(10)
  }
  triggers {
    issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
  }
  parameters {
    booleanParam(name: "SKIP_SCENARIOS", defaultValue: true, description: "If it's needed to skip those scenarios marked as @skip. Default true")
    booleanParam(name: "NIGHTLY_SCENARIOS", defaultValue: false, description: "If it's needed to include the scenarios marked as @nightly in the test execution. Default false")
    string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)')
    booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
    booleanParam(name: "forceSkipPresubmit", defaultValue: false, description: "If it's needed to execute the pre-submit tests: unit and precommit.")
    booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify to Slack with green builds.")
    string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
    string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
    string(name: 'BEAT_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
    string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.10-SNAPSHOT', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
    booleanParam(name: "BEATS_USE_CI_SNAPSHOTS", defaultValue: false, description: "If it's needed to use the binary snapshots produced by Beats CI instead of the official releases")
    choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'TRACE', 'INFO'], description: 'Log level to be used')
    choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
    string(name: 'STACK_VERSION', defaultValue: '8.0.0-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
    string(name: 'HELM_CHART_VERSION', defaultValue: '7.10.0', description: 'SemVer version of Helm chart to be used.')
    string(name: 'HELM_VERSION', defaultValue: '3.4.1', description: 'SemVer version of Helm to be used.')
    string(name: 'HELM_KIND_VERSION', defaultValue: '0.8.1', description: 'SemVer version of Kind to be used.')
    string(name: 'HELM_KUBERNETES_VERSION', defaultValue: '1.18.2', description: 'SemVer version of Kubernetes to be used.')
    string(name: 'GITHUB_CHECK_NAME', defaultValue: '', description: 'Name of the GitHub check to be updated. Only if this build is triggered from another parent stream.')
    string(name: 'GITHUB_CHECK_REPO', defaultValue: '', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.')
    string(name: 'GITHUB_CHECK_SHA1', defaultValue: '', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.')
  }
  stages {
    stage('Initializing'){
      agent { label 'ubuntu-18.04 && immutable && docker' }
      options { skipDefaultCheckout() }
      environment {
        HOME = "${env.WORKSPACE}"
        ELASTIC_APM_GLOBAL_LABELS="build_pr=${isPR()},build_id=${env.BUILD_ID}"
        PATH = "${env.PATH}:${env.WORKSPACE}/bin:${env.WORKSPACE}/${env.BASE_DIR}/.ci/scripts"
        GO111MODULE = 'on'
        SKIP_SCENARIOS = "${params.SKIP_SCENARIOS}"
        NIGHTLY_SCENARIOS = "${params.NIGHTLY_SCENARIOS}"
        SLACK_CHANNEL = "${params.SLACK_CHANNEL.trim()}"
        ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}"
        BEAT_VERSION = "${params.BEAT_VERSION.trim()}"
        BEATS_USE_CI_SNAPSHOTS = "${params.BEATS_USE_CI_SNAPSHOTS}"
        STACK_VERSION = "${params.STACK_VERSION.trim()}"
        FORCE_SKIP_GIT_CHECKS = "${params.forceSkipGitChecks}"
        FORCE_SKIP_PRESUBMIT = "${params.forceSkipPresubmit}"
        HELM_CHART_VERSION = "${params.HELM_CHART_VERSION.trim()}"
        HELM_VERSION = "${params.HELM_VERSION.trim()}"
        HELM_KIND_VERSION = "${params.HELM_KIND_VERSION.trim()}"
        HELM_KUBERNETES_VERSION = "${params.HELM_KUBERNETES_VERSION.trim()}"
        LOG_LEVEL = "${params.LOG_LEVEL.trim()}"
        TIMEOUT_FACTOR = "${params.TIMEOUT_FACTOR.trim()}"
      }
      stages {
        stage('Checkout') {
          steps {
            pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
            deleteDir()
            gitCheckout(basedir: BASE_DIR, githubNotifyFirstTimeContributor: true)
            githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
            stash allowEmpty: true, name: 'source', useDefaultExcludes: false
            setEnvVar("GO_VERSION", readFile("${env.WORKSPACE}/${env.BASE_DIR}/.go-version").trim())
            dir("${BASE_DIR}"){
              // Skip all the test stages for PR's with markdown changes only
              setEnvVar("SKIP_TESTS", isGitRegionMatch(patterns: [ '.*\\.md' ], shouldMatchAll: true))
            }
          }
        }
        stage('Pre-Submit') {
          when {
            beforeAgent true
            expression { return env.FORCE_SKIP_PRESUBMIT == "false" }
          }
          parallel {
            stage('Sanity checks') {
              agent { label 'ubuntu-18.04 && immutable && docker' }
              environment {
                PATH = "${env.WORKSPACE}/${env.BASE_DIR}/bin:${env.PATH}"
                GO111MODULE = 'auto'
              }
              options { skipDefaultCheckout() }
              steps {
                withGithubNotify(context: 'Sanity checks', tab: 'tests') {
                  deleteDir()
                  unstash 'source'
                  withGoEnv(version: "${GO_VERSION}"){
                    dir(BASE_DIR){
                      retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh script: '.ci/scripts/install-dependencies.sh', label: 'Install dependencies' }
                      preCommit(commit: "${GIT_BASE_COMMIT}", junit: true)
                    }
                  }
                }
              }
            }
            stage('Unit Tests') {
              agent { label 'ubuntu-18.04 && immutable && docker' }
              options { skipDefaultCheckout() }
              when {
                beforeAgent true
                expression { return env.SKIP_TESTS == "false" }
              }
              steps {
                withGithubNotify(context: 'Tests', tab: 'tests') {
                  deleteDir()
                  unstash 'source'
                  withGoEnv(version: "${GO_VERSION}"){
                    dir(BASE_DIR){
                      sh script: '.ci/scripts/build-test.sh', label: 'Build and test'
                    }
                  }
                }
              }
              post {
                always {
                  junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/outputs/TEST-unit-*.xml")
                  archiveArtifacts allowEmptyArchive: true, artifacts: "${BASE_DIR}/outputs/TEST-unit-*.xml"
                }
              }
            }
          }
        }
        stage('Build Docs') {
          options { skipDefaultCheckout() }
          when {
            beforeAgent true
            anyOf {
              expression { return env.FORCE_SKIP_GIT_CHECKS == "true" }
              expression { return env.SKIP_TESTS == "false" }
            }
          }
          steps {
            deleteDir()
            unstash 'source'
            dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
            dir("${BASE_DIR}/e2e") {
              sh(label: 'Build docs', script: 'make build-docs')
            }
          }
          post {
            always {
              dir("${BASE_DIR}") {
                archiveArtifacts allowEmptyArchive: true, artifacts: "e2e/docs/**"
              }
            }
          }
        }
        stage('End-To-End Tests') {
          failFast true
          options { skipDefaultCheckout() }
          environment {
            GO111MODULE = 'on'
            PATH = "${env.HOME}/bin:${env.WORKSPACE}/${env.BASE_DIR}/bin:${HOME}/go/bin:${env.PATH}"
          }
          when {
            beforeAgent true
            anyOf {
              expression { return env.FORCE_SKIP_GIT_CHECKS == "true" }
              expression { return env.SKIP_TESTS == "false" }
            }
          }
          steps {
            withGithubNotify(context: 'E2E Tests', tab: 'tests') {
              deleteDir()
              unstash 'source'
              dir("${BASE_DIR}") {
                script {
                  def suitesParam = params.runTestsSuites
                  def existingSuites = readYaml(file: '.ci/.e2e-tests.yaml')
                  def parallelTasks = [:]

                  if (suitesParam == "") {
                    log(level: 'DEBUG', text: "Iterate through existing test suites")
                    existingSuites['SUITES'].each { item ->
                      checkTestSuite(parallelTasks, item)
                    }
                  } else {
                    log(level: 'DEBUG', text: "Iterate through the comma-separated test suites (${suitesParam}), comparing with the existing test suites")
                    suitesParam.split(',').each { suiteParam ->
                      existingSuites['SUITES'].findAll { suiteParam.trim() == it.suite }.each { item ->
                        checkTestSuite(parallelTasks, item)
                      }
                    }
                  }
                  parallel(parallelTasks)
                }
              }
            }
          }
        }
        stage('Release') {
          options { skipDefaultCheckout() }
          when { tag "v*" }
          steps {
            deleteDir()
            unstash 'source'
            dir("${BASE_DIR}") {
              script {
                def matrix = readYaml(file: '.ci/.package.yaml')
                def parallelTasks = [:]
                matrix['OSS'].each { oss ->
                  matrix['PLATFORM'].each { platform ->
                    parallelTasks["${oss}-${platform}"] = generateStep(oss: "${oss}", platform: "${platform}")
                  }
                }
                parallel(parallelTasks)
              }
            }
          }
        }
      }
    }
  }
  post {
    cleanup {
      doNotifyBuildResult(params.notifyOnGreenBuilds)
    }
  }
}

def checkTestSuite(Map parallelTasks = [:], Map item = [:]) {
  def suite = item.suite
  def platforms = item.platforms
  item.scenarios.each { scenario ->
    def pullRequestFilter = scenario.containsKey('pullRequestFilter') ? scenario.pullRequestFilter : ''
    def tags = scenario.tags
    def regexps = [ "^e2e/_suites/${suite}/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go" ]
    if ("${FORCE_SKIP_GIT_CHECKS}" == "true" || isGitRegionMatch(patterns: regexps, shouldMatchAll: false)) {
      platforms.each { platform ->
        log(level: 'INFO', text: "Adding ${platform}:${suite}:${tags} test suite to the build execution")
        parallelTasks["${platform}_${suite}_${tags}"] = generateFunctionalTestStep(platform: "${platform}", suite: "${suite}", tags: "${tags}", pullRequestFilter: "${pullRequestFilter}")
      }
    } else {
      log(level: 'WARN', text: "The ${suite}:${tags} test suite won't be executed because there are no modified files")
    }
  }
}

def doNotifyBuildResult(boolean slackNotify) {
  githubCheckNotify(currentBuild.currentResult == 'SUCCESS' ? 'SUCCESS' : 'FAILURE')

  def testsSuites = "${params.runTestsSuites}"
  if (testsSuites?.trim() == "") {
    testsSuites = "All suites"
  }

  def channels = "${env.SLACK_CHANNEL}"
  if (channels?.trim() == "") {
    channels = "observablt-bots"
  }

  def header = "*Test Suite*: " + testsSuites
  notifyBuildResult(analyzeFlakey: true, flakyReportIdx: "reporter-e2e-tests-end-2-end-tests-pipeline-${env.JOB_BASE_NAME}", prComment: true, slackHeader: header, slackChannel: "${channels}", slackComment: true, slackNotify: slackNotify)
}


def generateStep(Map args = [:]){
  def oss = args.get('oss')
  def platform = args.get('platform')
  return {
    node('ubuntu-18.04 && immutable && docker') {
      try {
        deleteDir()
        unstash 'source'
        dir("${BASE_DIR}/cli") {
          withEnv(["GOOS=${oss}", "GOARCH=${platform}"]) {
            sh script: 'make build', label: 'Create releases'
          }
        }
      } catch(e) {
        error(e.toString())
      } finally {
        archiveArtifacts allowEmptyArchive: true, artifacts: "${BASE_DIR}/cli/.github/releases/download/**"
      }
    }
  }
}

def generateFunctionalTestStep(Map args = [:]){
  def platform = args.get('platform')
  def suite = args.get('suite')
  def tags = args.get('tags')
  def pullRequestFilter = args.get('pullRequestFilter')?.trim() ? args.get('pullRequestFilter') : ''

  // We will decide whether to include the nightly tests in the execution at CI time, only.
  // On the other hand, the developers can use the TAGS environment variable locally.
  // Finally, we positively know that tags are not empty, so we can use AND operator.
  def excludeNightlyTag = " && ~${NIGHTLY_TAG}"
  if ("${NIGHTLY_SCENARIOS}" == "true") {
    excludeNightlyTag = ""
  }
  tags += excludeNightlyTag

  if (isPR() || isUpstreamTrigger(filter: 'PR-')) {
    tags += pullRequestFilter
  }

  return {
    node("${platform} && immutable && docker") {
      try {
        deleteDir()
        unstash 'source'
        withGoEnv(version: "${GO_VERSION}"){
          if(isInstalled(tool: 'docker', flag: '--version')) {
            dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
          }
          filebeat(output: "docker_logs_${suite}_${tags}.log", workdir: "${env.WORKSPACE}"){
            dir("${BASE_DIR}"){
              withSecretVault(secret: "${VAULT_INSTRUMENTATION_SECRET}", user_key: "apmServerToken", user_var_name: "APM_SECRET_TOKEN", pass_key: "apmServerUrl", pass_var_name: "APM_SERVER_URL"){
                sh script: """.ci/scripts/functional-test.sh "${suite}" "${tags}" "${STACK_VERSION}" "${BEAT_VERSION}" """, label: "Run functional tests for ${suite}:${tags}"
              }
            }
          }
        }
      } catch(e) {
        error(e.toString())
      } finally {
        junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/outputs/TEST-*.xml")
          archiveArtifacts allowEmptyArchive: true, artifacts: "${BASE_DIR}/outputs/TEST-*.xml"
      }
    }
  }
}

/**
 Notify the GitHub check of the parent stream
**/
def githubCheckNotify(String status) {
  if (params.GITHUB_CHECK_NAME?.trim() && params.GITHUB_CHECK_REPO?.trim() && params.GITHUB_CHECK_SHA1?.trim()) {
    githubNotify context: "${params.GITHUB_CHECK_NAME}",
                 description: "${params.GITHUB_CHECK_NAME} ${status.toLowerCase()}",
                 status: "${status}",
                 targetUrl: "${env.RUN_DISPLAY_URL}",
                 sha: params.GITHUB_CHECK_SHA1, account: 'elastic', repo: params.GITHUB_CHECK_REPO, credentialsId: env.JOB_GIT_CREDENTIALS
  }
}