Jenkinsfile

#!/usr/bin/env groovy

@Library('apm@current') _

pipeline {
  agent { label 'ubuntu-20.04 && immutable && docker' }
  environment {
    REPO = 'e2e-testing'
    BASE_DIR = "src/github.com/elastic/${env.REPO}"
    ELASTIC_APM_ACTIVE="true"
    ELASTIC_APM_ENVIRONMENT="ci"
    ELASTIC_APM_LOG_FILE="stderr"
    ELASTIC_APM_LOG_LEVEL="debug"
    NIGHTLY_TAG="@nightly"
    NOTIFY_TO = credentials('notify-to')
    JOB_GCS_BUCKET = credentials('gcs-bucket')
    JOB_GIT_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken"
    DOCKER_ELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
    DOCKER_REGISTRY = 'docker.elastic.co'
    ELASTIC_CLOUD_SECRET = 'secret/observability-team/ci/elastic-cloud/observability-team-user'
    GCP_PROVISIONER_SECRET = 'secret/observability-team/ci/service-account/jenkins-gce-provisioner'
    AWS_PROVISIONER_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
    TEST_MATRIX_FILE = "${params.testMatrixFile}"
    RUN_ID = UUID.randomUUID().toString()
  }
  options {
    timeout(time: 1, unit: 'HOURS')
    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
    timestamps()
    ansiColor('xterm')
    disableResume()
    durabilityHint('PERFORMANCE_OPTIMIZED')
    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
    quietPeriod(10)
  }
  triggers {
    issueCommentTrigger("${obltGitHubComments()}")
  }
  parameters {
    booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.')
    booleanParam(name: "SKIP_SCENARIOS", defaultValue: true, description: "If it's needed to skip those scenarios marked as @skip. Default true")
    booleanParam(name: "NIGHTLY_SCENARIOS", defaultValue: false, description: "If it's needed to include the scenarios marked as @nightly in the test execution. Default false")
    string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)')
    string(name: 'testMatrixFile', defaultValue: '.ci/.e2e-tests.yaml', description: 'The file with the test suite and scenarios to be tested.')
    booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
    booleanParam(name: "forceSkipPresubmit", defaultValue: false, description: "If it's needed to execute the pre-submit tests: unit and precommit.")
    booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify to Slack with green builds.")
    string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
    string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
    string(name: 'BEAT_VERSION', defaultValue: '8.1.0-befff95a-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
    string(name: 'ELASTIC_AGENT_STALE_VERSION', defaultValue: '7.15-SNAPSHOT', description: 'SemVer version of the stale stand-alone elastic-agent to be used for Fleet upgrade tests.')
    choice(name: 'LOG_LEVEL', choices: ['DEBUG', 'TRACE', 'INFO'], description: 'Log level to be used')
    choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
    string(name: 'KIBANA_VERSION', defaultValue: '', description: 'Docker tag of the kibana to be used for the tests. It will refer to an image related to a Kibana PR, under the Observability-CI namespace')
    string(name: 'STACK_VERSION', defaultValue: '8.1.0-befff95a-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
    string(name: 'HELM_CHART_VERSION', defaultValue: '7.11.2', description: 'SemVer version of Helm chart to be used.')
    string(name: 'HELM_VERSION', defaultValue: '3.5.2', description: 'SemVer version of Helm to be used.')
    string(name: 'KIND_VERSION', defaultValue: '0.10.0', description: 'SemVer version of Kind to be used.')
    string(name: 'KUBERNETES_VERSION', defaultValue: '1.18.2', description: 'SemVer version of Kubernetes to be used.')
    string(name: 'GITHUB_CHECK_NAME', defaultValue: '', description: 'Name of the GitHub check to be updated. Only if this build is triggered from another parent stream.')
    string(name: 'GITHUB_CHECK_REPO', defaultValue: '', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.')
    string(name: 'GITHUB_CHECK_SHA1', defaultValue: '', description: 'Git SHA for the Beats upstream project (branch or PR)')
  }
  stages {
    stage('Initializing'){
      options { skipDefaultCheckout() }
      environment {
        HOME = "${env.WORKSPACE}"
        PATH = "${env.PATH}:${env.WORKSPACE}/bin:${env.WORKSPACE}/${env.BASE_DIR}/.ci/scripts"
        GO111MODULE = 'on'
        SKIP_SCENARIOS = "${params.SKIP_SCENARIOS}"
        NIGHTLY_SCENARIOS = "${params.NIGHTLY_SCENARIOS}"
        SLACK_CHANNEL = "${params.SLACK_CHANNEL.trim()}"
        ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}"
        BEAT_VERSION = "${params.BEAT_VERSION.trim()}"
        KIBANA_VERSION = "${params.KIBANA_VERSION.trim()}"
        STACK_VERSION = "${params.STACK_VERSION.trim()}"
        FORCE_SKIP_GIT_CHECKS = "${params.forceSkipGitChecks}"
        FORCE_SKIP_PRESUBMIT = "${params.forceSkipPresubmit}"
        HELM_CHART_VERSION = "${params.HELM_CHART_VERSION.trim()}"
        HELM_VERSION = "${params.HELM_VERSION.trim()}"
        KIND_VERSION = "${params.KIND_VERSION.trim()}"
        KUBERNETES_VERSION = "${params.KUBERNETES_VERSION.trim()}"
        LOG_LEVEL = "${params.LOG_LEVEL.trim()}"
        TIMEOUT_FACTOR = "${params.TIMEOUT_FACTOR.trim()}"
      }
      stages {
        stage('Checkout') {
          steps {
            pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
            deleteDir()
            gitCheckout(basedir: BASE_DIR, githubNotifyFirstTimeContributor: true)
            githubCheckNotify('PENDING')  // we want to notify the upstream about the e2e the soonest
            stash allowEmpty: true, name: 'source', useDefaultExcludes: false
            setEnvVar("GO_VERSION", readFile("${env.WORKSPACE}/${env.BASE_DIR}/.go-version").trim())
            checkSkipTests()
          }
        }
        stage('Pre-Submit') {
          when {
            beforeAgent true
            expression { return env.FORCE_SKIP_PRESUBMIT == "false" }
          }
          parallel {
            stage('Sanity checks') {
              agent { label 'ubuntu-20.04 && immutable && docker' }
              environment {
                PATH = "${env.WORKSPACE}/${env.BASE_DIR}/bin:${env.PATH}"
                GO111MODULE = 'auto'
              }
              options { skipDefaultCheckout() }
              steps {
                withGithubNotify(context: 'Sanity checks', tab: 'tests') {
                  deleteDir()
                  unstash 'source'
                  withGoEnv(version: "${GO_VERSION}"){
                    dir(BASE_DIR){
                      retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh script: '.ci/scripts/install-dependencies.sh', label: 'Install dependencies' }
                      preCommit(commit: "${GIT_BASE_COMMIT}", junit: true)
                    }
                  }
                }
              }
            }
            stage('Unit Tests') {
              options { skipDefaultCheckout() }
              when {
                beforeAgent true
                anyOf {
                  expression { return env.FORCE_SKIP_GIT_CHECKS == "true" }
                  expression { return env.SKIP_TESTS == "false" }
                }
              }
              steps {
                withGithubNotify(context: 'Tests', tab: 'tests') {
                  deleteDir()
                  unstash 'source'
                  withGoEnv(version: "${GO_VERSION}"){
                    dir(BASE_DIR){
                      sh script: '.ci/scripts/build-test.sh', label: 'Build and test'
                    }
                  }
                }
              }
              post {
                always {
                  junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/outputs/TEST-unit-*.xml")
                  archiveArtifacts allowEmptyArchive: true, artifacts: "${BASE_DIR}/outputs/TEST-unit-*.xml"
                }
              }
            }
          }
        }
        stage('Build Docs') {
          options { skipDefaultCheckout() }
          when {
            beforeAgent true
            anyOf {
              expression { return env.FORCE_SKIP_GIT_CHECKS == "true" }
              expression { return env.SKIP_TESTS == "false" }
            }
          }
          steps {
            deleteDir()
            unstash 'source'
            dockerLogin(secret: "${DOCKER_ELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
            dir("${BASE_DIR}/e2e") {
              sh(label: 'Build docs', script: 'make build-docs')
            }
          }
          post {
            always {
              dir("${BASE_DIR}") {
                archiveArtifacts allowEmptyArchive: true, artifacts: "e2e/docs/**"
              }
            }
          }
        }

        stage('Deploy Test Infra') {
          failFast true
          options { skipDefaultCheckout() }
          environment {
            GO111MODULE = 'on'
            PATH = "${env.HOME}/bin:${env.WORKSPACE}/${env.BASE_DIR}/bin:${HOME}/go/bin:${env.PATH}"
          }
          when {
            beforeAgent true
            anyOf {
              expression { return env.FORCE_SKIP_GIT_CHECKS == "true" }
              expression { return env.SKIP_TESTS == "false" }
            }
          }
          steps {
            withGithubNotify(context: 'Deploy Stack', tab: 'tests') {
              deleteDir()
              unstash 'source'
              dir("${BASE_DIR}") {
                script {
                  def stackWorkspace = "${env.WORKSPACE}/${env.BASE_DIR}"
                  // Deploy the test infrastructure
                  sh "ssh-keygen -b 4096 -t rsa -f ${stackWorkspace}/e2essh -q -N \"\" "
                  def stackMachine = getMachineInfo(stackWorkspace, 'stack')

                  ansible(stackWorkspace,
                          env.RUN_ID.split('-')[0],
                          "-t provision-stack --extra-vars=\"nodeLabel=stack nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\"")

                  // Must be gathered after deployment as the public IP is known at that time
                  def stackRunner = getNodeIp(stackWorkspace, 'stack')

                  ansible(stackWorkspace,
                          env.RUN_ID.split('-')[0],
                          "-i \"${stackRunner.ip},\" -t setup-stack --extra-vars=\"nodeLabel=stack nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\"")

                  // Update stash with latest changes from the stack deployment
                  stash allowEmpty: true, name: 'sourceEnvModified', useDefaultExcludes: false
                }
              }
            }
          }
        }

        stage('End-To-End Tests') {
          failFast true
          options { skipDefaultCheckout() }
          environment {
            GO111MODULE = 'on'
            PATH = "${env.HOME}/bin:${env.WORKSPACE}/${env.BASE_DIR}/bin:${HOME}/go/bin:${env.PATH}"
          }
          when {
            beforeAgent true
            anyOf {
              expression { return env.FORCE_SKIP_GIT_CHECKS == "true" }
              expression { return env.SKIP_TESTS == "false" }
            }
          }
          steps {
            withGithubNotify(context: 'E2E Tests', tab: 'tests') {
              script {
                def suitesParam = params.runTestsSuites
                def existingSuites = readYaml(file: "${env.WORKSPACE}/${env.BASE_DIR}/${TEST_MATRIX_FILE}")
                def parallelTasks = [:]
                if (suitesParam == "") {
                  log(level: 'DEBUG', text: "Iterate through existing test suites")
                  existingSuites['SUITES'].each { item ->
                    checkTestSuite(parallelTasks, item)
                  }
                } else {
                  log(level: 'DEBUG', text: "Iterate through the comma-separated test suites (${suitesParam}), comparing with the existing test suites")
                  suitesParam.split(',').each { suiteParam ->
                    existingSuites['SUITES'].findAll { suiteParam.trim() == it.suite }.each { item ->
                      checkTestSuite(parallelTasks, item)
                    }
                  }
                }
                parallel(parallelTasks)
              }
            }
          }
          post {
            cleanup {
              // Once all tests are complete we need to teardown the single instance with the deployed stack
              script {
                def stackWorkspace = "${env.WORKSPACE}/${env.BASE_DIR}"
                def stackMachine = getMachineInfo(stackWorkspace, 'stack')
                ansible(stackWorkspace,
                        env.RUN_ID.split('-')[0],
                        "-t destroy --extra-vars=\"nodeLabel=stack nodeImage=${stackMachine.image} nodeInstanceType=${stackMachine.instance_type}\"")
              }
            }
          }
        }
        stage('Release') {
          options { skipDefaultCheckout() }
          when { tag "v*" }
          steps {
            deleteDir()
            unstash 'source'
            dir("${BASE_DIR}") {
              setEnvVar("GITHUB_TOKEN", getGithubToken())
              retryWithSleep(retries: 2, seconds: 5, backoff: true) {
                sh(label: 'Release binaries with gorelease', script: 'curl -sL https://git.io/goreleaser | bash -s -- --rm-dist', returnStatus: true)
              }
            }
          }
          post {
            always {
              archiveArtifacts allowEmptyArchive: true, artifacts: "${BASE_DIR}/cli/dist/**"
            }
          }
        }
      }
    }
  }
  post {
    cleanup {
      doNotifyBuildResult(params.notifyOnGreenBuilds)
    }
  }
}

// this function evaluates if the test stage of the build must be executed
def checkSkipTests() {
  dir("${BASE_DIR}"){

    // only docs means no tests are run
    if (isGitRegionMatch(patterns: [ '.*\\.md' ], shouldMatchAll: true)) {
      setEnvVar("SKIP_TESTS", true)
      return
    }

    // patterns for all places that should trigger a full build
    def regexps = [ "^e2e/_suites/fleet/.*", "^e2e/_suites/helm/.*", "^e2e/_suites/kubernetes-autodiscover/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go", "^internal/.*\\.go" ]
    setEnvVar("SKIP_TESTS", !isGitRegionMatch(patterns: regexps, shouldMatchAll: false))
  }
}


def sshexec(workspace, connection, cmd){
  sh "ssh -tt -o TCPKeepAlive=yes -o ServerAliveInterval=60 -o ServerAliveCountMax=10 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${workspace}/e2essh ${connection.user}@${connection.ip} -- '${cmd}'"
}

def scpr(workspace, connection, remote_src, local_dst){
  sh "scp -r -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i ${workspace}/e2essh ${connection.user}@${connection.ip}:${remote_src} ${local_dst}"
}

/*
 * Runs python in venv
 */
def pyrun(command){
  def awsProps = getVaultSecret(secret: "${AWS_PROVISIONER_SECRET}")
  def awsAuthObj = awsProps?.data
  withEnv([
    "ANSIBLE_HOST_KEY_CHECKING=False",
  ]){
    withVaultToken(){
      withEnvMask(vars: [
        [var: "AWS_ACCESS_KEY_ID", password: awsAuthObj.access_key],
        [var: "AWS_SECRET_ACCESS_KEY", password: awsAuthObj.secret_key]
      ]) {
        sh(script: """#!/bin/bash
            set -eux
            if ! test -d ${env.WORKSPACE}/.venv; then
                python3 -m venv ${env.WORKSPACE}/.venv
                 ${env.WORKSPACE}/.venv/bin/pip3 -q install wheel
                 ${env.WORKSPACE}/.venv/bin/pip3 -q install ansible requests boto3 boto
                 ${env.WORKSPACE}/.venv/bin/ansible-galaxy install -r .ci/ansible/requirements.yml
            fi
            ${env.WORKSPACE}/.venv/bin/${command}
          """,
           label: "Executing ${command.split(' ')[0]}"
        )
      }
    }
  }
}

def getNodeIp(workspace, platform){
  def runners = readYaml(file: "${workspace}/${platform}-sshhosts")
  def runnerUser = runners[0].split('@')[0]
  def runnerIP = runners[0].split('@')[1]
  return [user: runnerUser, ip: runnerIP]
}

def getMachineInfo(workspace, platform){
  def machineYaml = readYaml(file: "${workspace}/${TEST_MATRIX_FILE}")
  def machines = machineYaml['PLATFORMS']
  return machines.get(platform)
}

def ansible(workspace, run_id, args){
     pyrun("ansible-playbook --private-key=\"${workspace}/e2essh\" --extra-vars=\"workspace=${workspace}/ runId=${run_id} sshPublicKey=${workspace}/e2essh.pub\" --ssh-common-args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' ${workspace}/.ci/ansible/playbook.yml ${args}")
}


def checkTestSuite(Map parallelTasks = [:], Map item = [:]) {
  def suite = item.suite
  def platforms = item.platforms

  item.scenarios.each { scenario ->
    def name = scenario.name
    def platformsValue = platforms
    def scenarioPlatforms = scenario.platforms
    if (scenarioPlatforms?.size() > 0) {
      // scenario platforms take precedence over suite platforms, overriding them
      platformsValue = scenarioPlatforms
    }
    def pullRequestFilter = scenario.containsKey('pullRequestFilter') ? scenario.pullRequestFilter : ''
    def tags = scenario.tags
      platformsValue.each { platform ->
        log(level: 'INFO', text: "Adding ${platform}:${suite}:${tags} test suite to the build execution")
        machineInfo = getMachineInfo("${env.WORKSPACE}/${env.BASE_DIR}", platform)
        parallelTasks["${platform}_${tags}"] = generateFunctionalTestStep(name: "${name}",
                                                                          platform: platform,
                                                                          suite: "${suite}",
                                                                          tags: "${tags}",
                                                                          pullRequestFilter: "${pullRequestFilter}",
                                                                          machine: machineInfo)
      }
  }
}

/*
 * Sends out notification of the build result to Slack
 */
def doNotifyBuildResult(boolean slackNotify) {
  githubCheckNotify(currentBuild.currentResult == 'SUCCESS' ? 'SUCCESS' : 'FAILURE')

  def testsSuites = "${params.runTestsSuites}"
  if (testsSuites?.trim() == "") {
    testsSuites = "All suites"
  }

  def channels = "${env.SLACK_CHANNEL}"
  if (channels?.trim() == "") {
    channels = "observablt-bots"
  }

  def header = "*Test Suite*: " + testsSuites
  notifyBuildResult(analyzeFlakey: true,
                    jobName: getFlakyJobName(withBranch: "${env.JOB_BASE_NAME}"),
                    prComment: true,
                    slackHeader: header,
                    slackChannel: "${channels}",
                    slackComment: true,
                    slackNotify: slackNotify)
}

/**
 Notify the GitHub check of the parent stream
 **/
def githubCheckNotify(String status) {
  if (params.GITHUB_CHECK_NAME?.trim() && params.GITHUB_CHECK_REPO?.trim() && params.GITHUB_CHECK_SHA1?.trim()) {
    githubNotify context: "${params.GITHUB_CHECK_NAME}",
      description: "${params.GITHUB_CHECK_NAME} ${status.toLowerCase()}",
      status: "${status}",
      targetUrl: "${env.RUN_DISPLAY_URL}",
      sha: params.GITHUB_CHECK_SHA1, account: 'elastic', repo: params.GITHUB_CHECK_REPO, credentialsId: env.JOB_GIT_CREDENTIALS
  }
}

def generateFunctionalTestStep(Map args = [:]){
  def name = args.get('name')
  def name_normalize = name.replace(' ', '_')
  def platform = args.get('platform')
  def suite = args.get('suite')
  def tags = args.get('tags')
  def pullRequestFilter = args.get('pullRequestFilter')?.trim() ? args.get('pullRequestFilter') : ''
  def machine = args.get('machine')

  // We will decide whether to include the nightly tests in the execution at CI time, only.
  // On the other hand, the developers can use the TAGS environment variable locally.
  // Finally, we positively know that tags are not empty, so we can use AND operator.
  def excludeNightlyTag = " && ~${NIGHTLY_TAG}"
  if ("${NIGHTLY_SCENARIOS}" == "true") {
    excludeNightlyTag = ""
  }
  tags += excludeNightlyTag

  // TODO: Is this still relevant?
  if (isPR() || isUpstreamTrigger(filter: 'PR-')) {
    // when the "Run_As_Master_Branch" param is disabled, we will honour the PR filters, which
    // basically exclude some less frequent platforms or operative systems. If the user enabled
    // this param, the pipeline will remove the filters from the test runner.
    if (!params.Run_As_Master_Branch) {
      tags += pullRequestFilter
    }
  }

  def goArch = "amd64"
  if (platform.contains("arm64")) {
    goArch = "arm64"
  }

  // Skip scenarios per platform
  tags += " && ~@skip:${goArch}"

  // Setup environment for platform
  def envContext = []
  // envContext.add("PROVIDER=${platformProvider}")
  envContext.add("SUITE=${suite}")
  envContext.add("ELASTIC_APM_GLOBAL_LABELS=branch_name=${BRANCH_NAME},build_pr=${isPR()},build_id=${env.BUILD_ID},go_arch=${goArch},beat_version=${env.BEAT_VERSION},stack_version=${env.STACK_VERSION}")

  def stackRunner = getNodeIp("${env.WORKSPACE}/${env.BASE_DIR}", 'stack')

  def runId = UUID.randomUUID().toString().split('-')[0]

  return {
    withNode(labels: 'ubuntu-20.04', forceWorkspace: true, forceWorker: true){
      try {
        deleteDir()
        unstash 'sourceEnvModified'
        withEnv(envContext) {
          // This step will help to send the APM traces to the
          // APM service defined by the Otel Jenkins plugin.
          // TODO: This will need to be integrated into the provisioned VMs
          withOtelEnv() {
            // Start node, capture ip address
            ansible("${env.WORKSPACE}",
                    runId,
                    "-t start-node --extra-vars=\"stackRunner=${stackRunner.ip} nodeLabel=${platform} nodeImage=${machine.image} nodeInstanceType=${machine.instance_type}\"")

            def testRunner = getNodeIp("${env.WORKSPACE}", platform)

            // Configure node for testing
            ansible("${env.WORKSPACE}",
                    runId,
                    "-i \"${testRunner.ip},\" -t setup-node --extra-vars=\"stackRunner=${stackRunner.ip} nodeLabel=${platform} nodeImage=${machine.image} nodeInstanceType=${machine.instance_type}\"")

            sshexec("${env.WORKSPACE}",
                    testRunner,
                    """sudo bash /home/${testRunner.user}/e2e-testing/.ci/scripts/functional-test.sh "${tags}" """)
          }
        }
      } finally {
        def testRunner = getNodeIp("${env.WORKSPACE}", platform)
        sh "mkdir -p outputs/${testRunner.ip} || true"
        sshexec("${env.WORKSPACE}",
                testRunner,
                """sudo chown ${testRunner.user}:${testRunner.user} -R /home/${testRunner.user}/e2e-testing/outputs """)
        scpr("${env.WORKSPACE}",
             testRunner,
             "e2e-testing/outputs/TEST-*${runId}*.xml",
             "outputs/${testRunner.ip}/.")
        sh "ls -l outputs/${testRunner.ip}"
        ansible("${env.WORKSPACE}",
                runId,
                "-t destroy --extra-vars=\"nodeLabel=${platform} nodeImage=${machine.image} nodeInstanceType=${machine.instance_type}\"")
        junit allowEmptyResults: true,
          keepLongStdio: true,
          testResults: "outputs/${testRunner.ip}/TEST-*${runId}*.xml"
        archiveArtifacts allowEmptyArchive: true,
          artifacts: "outputs/${testRunner.ip}/TEST-*${runId}*.xml"
      }
    }
  }
}