From df1ebf73fb88cde487de6f03ec630bd3e8b751d3 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 24 Mar 2020 10:07:08 -0400 Subject: [PATCH] [ci] Make tests pass in Jenkinsfile pipeline (#17071) * Address failing tests and warnings * Ensure mage unitTest works consistently Windows will invoke `mage unitTest` that will run the Go and Python tests without requiring the integration tests environment. Use an alias for goTestUnit in metricbeat * Only include test output when `mage -v` is used The full test output is written to a file and we archive that. Additionally if a test fails that output is still logged out stderr. This helps speed things up because Jenkins doesn't need to deal with streaming megabytes of log output to the console. It also makes it easier to spot errors in the log. * Jenkinsfile updates Changes: - Disable macOS by default. These workers don't have python3 so they cannot a portion of the tests. - Add python38 to the PATH on Windows. - Various fixes to the environment variables on Windows. - Always add the GVM GOROOT/bin to the PATH on POSIX. Same as was being done on Windows. - Use bat instead of sh on Windows to run `mage dumpVariables`. * Add MAGEFILE_VERBOSE to travis-ci env --- .ci/scripts/install-go.sh | 2 +- .ci/scripts/install-tools.bat | 13 +- .editorconfig | 4 + .travis.yml | 3 + Jenkinsfile | 183 ++++++++++++------ Vagrantfile | 6 +- auditbeat/magefile.go | 27 +-- dev-tools/mage/gotest.go | 9 +- dev-tools/mage/target/integtest/integtest.go | 7 +- filebeat/magefile.go | 25 +-- filebeat/tests/system/test_tcp.py | 2 +- metricbeat/magefile.go | 26 +-- .../system/diskio/diskstat_windows_test.go | 3 + winlogbeat/magefile.go | 1 + x-pack/auditbeat/magefile.go | 25 +-- x-pack/dockerlogbeat/magefile.go | 7 +- x-pack/filebeat/magefile.go | 25 +-- x-pack/functionbeat/magefile.go | 2 + x-pack/functionbeat/tests/system/test_base.py | 12 +- x-pack/libbeat/docker-compose.yml | 7 +- x-pack/libbeat/magefile.go | 15 +- x-pack/metricbeat/magefile.go | 26 +-- x-pack/winlogbeat/magefile.go | 2 + 23 files changed, 237 insertions(+), 195 deletions(-) diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh index 6698c1f85a23..5af9f338ca14 100755 --- a/.ci/scripts/install-go.sh +++ b/.ci/scripts/install-go.sh @@ -10,7 +10,7 @@ GVM_CMD="${HOME}/bin/gvm" mkdir -p "${HOME}/bin" -curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-${ARCH}-amd64" +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64" chmod +x "${GVM_CMD}" gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat index 0f48d28bda39..54bcaf5e8a44 100644 --- a/.ci/scripts/install-tools.bat +++ b/.ci/scripts/install-tools.bat @@ -9,8 +9,19 @@ IF ERRORLEVEL 1 ( mkdir %WORKSPACE%\bin where /q gvm IF ERRORLEVEL 1 ( - curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-windows-amd64.exe + curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe ) FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i +go env go install -mod=vendor github.com/magefile/mage +mage -version +where mage + +if not exist C:\Python38\python.exe ( + REM Install python 3.8. + choco install python -y -r --no-progress --version 3.8.2 +) +python --version +where python + diff --git a/.editorconfig b/.editorconfig index ff8d698c9a8a..16a560fec2b2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -29,6 +29,10 @@ indent_style = tab [*.mk] indent_style = tab +[Jenkinsfile] +indent_size = 2 +indent_style = space + [Vagrantfile] indent_size = 2 indent_style = space diff --git a/.travis.yml b/.travis.yml index ddeb794dc2c3..a06aca007924 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,9 @@ env: # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704 - TRAVIS_MINIKUBE_VERSION=v0.25.2 - MACOSX_DEPLOYMENT_TARGET=10.15 + # Enable verbose output since Travis users cannot access test output that is written to files. + # The constant output from tests also prevents time-outs. + - MAGEFILE_VERBOSE=true # Only run CI jobs on specific branches. # To keep the number of Travis CI jobs in check, we decided to configure Travis to only kick off CI jobs for the master branch. diff --git a/Jenkinsfile b/Jenkinsfile index 1087f551050b..3533e905ee23 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -25,6 +25,7 @@ pipeline { parameters { booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') } stages { @@ -77,7 +78,7 @@ pipeline { } } steps { - makeTarget("Filebeat x-pack Linux", "-C x-pack/filebeat testsuite") + mageTarget("Filebeat x-pack Linux", "x-pack/filebeat", "update build test") } } stage('Filebeat Mac OS X'){ @@ -86,7 +87,7 @@ pipeline { when { beforeAgent true expression { - return env.BUILD_FILEBEAT != "false" + return env.BUILD_FILEBEAT != "false" && params.macosTest } } steps { @@ -103,8 +104,7 @@ pipeline { } } steps { - mageTargetWin("Filebeat oss Windows Unit test", "-d filebeat goUnitTest") - //mageTargetWin("Filebeat oss Windows Integration test", "-d filebeat goIntegTest") + mageTargetWin("Filebeat oss Windows Unit test", "filebeat", "unitTest") } } stage('Heartbeat'){ @@ -125,6 +125,12 @@ pipeline { stage('Heartbeat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } steps { makeTarget("Heartbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C heartbeat testsuite") } @@ -139,7 +145,7 @@ pipeline { } } steps { - mageTargetWin("Heartbeat oss Windows Unit test", "-d heartbeat goTestUnit") + mageTargetWin("Heartbeat oss Windows Unit test", "heartbeat", "unitTest") } } } @@ -167,6 +173,12 @@ pipeline { stage('Auditbeat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } steps { makeTarget("Auditbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C auditbeat testsuite") } @@ -181,8 +193,7 @@ pipeline { } } steps { - mageTargetWin("Auditbeat Windows Unit test", "-d auditbeat goUnitTest") - //mageTargetWin("Auditbeat Windows Integration test", "-d auditbeat goIntegTest") + mageTargetWin("Auditbeat Windows Unit test", "auditbeat", "unitTest") } } } @@ -197,7 +208,7 @@ pipeline { } } steps { - makeTarget("Auditbeat x-pack Linux", "-C x-pack/auditbeat testsuite") + mageTarget("Auditbeat x-pack Linux", "x-pack/auditbeat", "update build test") } } stage('Libbeat'){ @@ -289,7 +300,7 @@ pipeline { } } steps { - makeTarget("Metricbeat x-pack Linux", "-C x-pack/metricbeat testsuite") + mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "update build test") } } stage('Metricbeat crosscompile'){ @@ -311,7 +322,7 @@ pipeline { when { beforeAgent true expression { - return env.BUILD_METRICBEAT != "false" + return env.BUILD_METRICBEAT != "false" && params.macosTest } } steps { @@ -328,8 +339,7 @@ pipeline { } } steps { - mageTargetWin("Metricbeat Windows Unit test", "-d metricbeat goUnitTest") - //mageTargetWin("Metricbeat Windows Integration test", "-d metricbeat goIntegTest") + mageTargetWin("Metricbeat Windows Unit test", "metricbeat", "unitTest") } } stage('Packetbeat'){ @@ -361,7 +371,7 @@ pipeline { stages { stage('Dockerlogbeat'){ steps { - makeTarget("Elastic Log Plugin unit tests", "-C x-pack/dockerlogbeat testsuite") + mageTarget("Elastic Docker Logging Driver Plugin unit tests", "x-pack/dockerlogbeat", "update build test") } } } @@ -391,7 +401,7 @@ pipeline { } } steps { - mageTargetWin("Winlogbeat Windows Unit test", "-d winlogbeat goUnitTest") + mageTargetWin("Winlogbeat Windows Unit test", "winlogbeat", "unitTest") } } } @@ -406,7 +416,7 @@ pipeline { } } steps { - mageTargetWin("Winlogbeat Windows Unit test", "-d x-pack/winlogbeat update:fields goUnitTest") + mageTargetWin("Winlogbeat Windows Unit test", "x-pack/winlogbeat", "unitTest") } } stage('Functionbeat'){ @@ -421,14 +431,20 @@ pipeline { stages { stage('Functionbeat x-pack'){ steps { - makeTarget("Functionbeat x-pack Linux", "-C x-pack/functionbeat testsuite") + mageTarget("Functionbeat x-pack Linux", "x-pack/functionbeat", "update build test") } } stage('Functionbeat Mac OS X x-pack'){ agent { label 'macosx' } options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } steps { - makeTarget("Functionbeat x-pack Mac OS X", "TEST_ENVIRONMENT=0 -C x-pack/functionbeat testsuite") + mageTarget("Functionbeat x-pack Mac OS X", "x-pack/functionbeat", "update build test") } } stage('Functionbeat Windows'){ @@ -441,7 +457,7 @@ pipeline { } } steps { - mageTargetWin("Functionbeat Windows Unit test", "-d x-pack/functionbeat goUnitTest") + mageTargetWin("Functionbeat Windows Unit test", "x-pack/functionbeat", "unitTest") } } } @@ -488,6 +504,12 @@ pipeline { stage('Generators Metricbeat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } steps { makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") } @@ -495,6 +517,12 @@ pipeline { stage('Generators Beat Mac OS X'){ agent { label 'macosx' } options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } steps { makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") } @@ -530,82 +558,108 @@ pipeline { } } -def makeTarget(context, target, clean = true){ - whenTrue(params.debug){ - dumpFilteredEnvironment() - dumpMage() - } +def makeTarget(String context, String target, boolean clean = true) { withGithubNotify(context: "${context}") { - withBeatsEnv(){ - sh(label: "Make ${target}", script: """ - eval "\$(gvm use ${GO_VERSION} --format=bash)" - make ${target} - """) - if(clean) { + withBeatsEnv(true) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + sh(label: "Make ${target}", script: "make ${target}") + if (clean) { sh(script: 'script/fix_permissions.sh ${HOME}') } } } } -def mageTargetWin(context, target){ - whenTrue(params.debug){ - dumpFilteredEnvironment() - withBeatsEnvWin(){ - dumpMage() +def mageTarget(String context, String directory, String target) { + withGithubNotify(context: "${context}") { + withBeatsEnv(true) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } } } +} + +def mageTargetWin(String context, String directory, String target) { withGithubNotify(context: "${context}") { - withBeatsEnvWin(){ - bat(label: "Mage ${target}", script: """ - set - mage ${target} - """) + withBeatsEnvWin() { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMageWin() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } } } } -def withBeatsEnv(Closure body){ +def withBeatsEnv(boolean archive, Closure body) { + def os = goos() + def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + withEnv([ "HOME=${env.WORKSPACE}", "GOPATH=${env.WORKSPACE}", - "PATH+GO=${env.WORKSPACE}/bin:${env.PATH}", - "MAGEFILE_CACHE=${WORKSPACE}\\.magefile", + "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", + "MAGEFILE_CACHE=${WORKSPACE}/.magefile", "TEST_COVERAGE=true", "RACE_DETECTOR=true", "PYTHON_ENV=${WORKSPACE}/python-env", "TEST_TAGS=oracle", "DOCKER_PULL=0", - ]){ + ]) { deleteDir() unstash 'source' - dir("${BASE_DIR}"){ + dir("${env.BASE_DIR}") { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") + sh(label: "Install Mage", script: "make mage") try { body() } finally { + if (archive) { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') + } + } reportCoverage() } } } } -def withBeatsEnvWin(Closure body){ +def withBeatsEnvWin(Closure body) { + final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + withEnv([ - "HOME=${WORKSPACE}", - "GOPATH=${WORKSPACE}", - "PATH+GO=${WORKSPACE}\\bin;${goRoot}\\bin;C:\\ProgramData\\chocolatey\\bin", - "MAGEFILE_CACHE=${WORKSPACE}\\.magefile", + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}", + "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", "TEST_COVERAGE=true", "RACE_DETECTOR=true", ]){ deleteDir() unstash 'source' - dir("${BASE_DIR}"){ - bat(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") + dir("${env.BASE_DIR}"){ + bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") try { body() } finally { @@ -618,14 +672,37 @@ def withBeatsEnvWin(Closure body){ } } +def goos(){ + def labels = env.NODE_LABELS + + if (labels.contains('linux')) { + return 'linux' + } else if (labels.contains('windows')) { + return 'windows' + } else if (labels.contains('darwin')) { + return 'darwin' + } + + throw new IllegalArgumentException("Unhandled OS name in NODE_LABELS: " + labels) +} + def dumpMage(){ echo "### MAGE DUMP ###" sh(label: "Dump mage variables", script: "mage dumpVariables") echo "### END MAGE DUMP ###" } +def dumpMageWin(){ + echo "### MAGE DUMP ###" + bat(label: "Dump mage variables", script: "mage dumpVariables") + echo "### END MAGE DUMP ###" +} + def dumpFilteredEnvironment(){ echo "### ENV DUMP ###" + echo "PATH: ${env.PATH}" + echo "HOME: ${env.HOME}" + echo "USERPROFILE: ${env.USERPROFILE}" echo "BUILD_DIR: ${env.BUILD_DIR}" echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" echo "BEATS: ${env.BEATS}" @@ -679,7 +756,7 @@ def k8sTest(versions){ stage("k8s ${v}"){ withEnv(["K8S_VERSION=${v}"]){ withGithubNotify(context: "K8s ${v}") { - withBeatsEnv(){ + withBeatsEnv(false) { sh(label: "Install k8s", script: """ eval "\$(gvm use ${GO_VERSION} --format=bash)" .ci/scripts/kind-setup.sh @@ -695,8 +772,6 @@ def k8sTest(versions){ def reportCoverage(){ catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**/TEST-*.xml") - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/TEST-*.out') retry(2){ sh(label: 'Report to Codecov', script: ''' curl -sSLo codecov https://codecov.io/bash diff --git a/Vagrantfile b/Vagrantfile index eba786adda20..fdff82a57ea1 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -44,7 +44,7 @@ if (-Not (Test-Path $gopath_beats)) { if (-Not (Get-Command "gvm" -ErrorAction SilentlyContinue)) { echo "Installing gvm to manage go version" [Net.ServicePointManager]::SecurityProtocol = "tls12" - Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe + Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe C:\\Windows\\System32\\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression go version @@ -83,7 +83,7 @@ choco feature disable -n=showDownloadProgress if (-Not (Get-Command "python" -ErrorAction SilentlyContinue)) { echo "Installing python 3" - choco install python -y -r --version 3.8.1.20200110 + choco install python -y -r --version 3.8.2 refreshenv $env:PATH = "$env:PATH;C:\\Python38;C:\\Python38\\Scripts" } @@ -119,7 +119,7 @@ def linuxGvmProvision(arch="amd64") return <