From fe732d27ff84c449a2cf3bb61e8376575411595e Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 25 Mar 2020 15:26:44 -0400 Subject: [PATCH] [ci] Backport Jenkinsfile and supporting changes (#17225) * Backport Jenkinsfile and supporting scripts Files are from b82b9dbb135c3df995ec2a1572b2c0a8f83de423 * [ci] Make tests pass in Jenkinsfile pipeline (#17071) * Address failing tests and warnings * Ensure mage unitTest works consistently Windows will invoke `mage unitTest` that will run the Go and Python tests without requiring the integration tests environment. Use an alias for goTestUnit in metricbeat * Only include test output when `mage -v` is used The full test output is written to a file and we archive that. Additionally if a test fails that output is still logged out stderr. This helps speed things up because Jenkins doesn't need to deal with streaming megabytes of log output to the console. It also makes it easier to spot errors in the log. * Jenkinsfile updates Changes: - Disable macOS by default. These workers don't have python3 so they cannot a portion of the tests. - Add python38 to the PATH on Windows. - Various fixes to the environment variables on Windows. - Always add the GVM GOROOT/bin to the PATH on POSIX. Same as was being done on Windows. - Use bat instead of sh on Windows to run `mage dumpVariables`. * Add MAGEFILE_VERBOSE to travis-ci env (cherry picked from commit 0a171b51a3bfb289714a80eb71d639840138ad57) --- .ci/scripts/install-docker-compose.sh | 13 + .ci/scripts/install-go.sh | 18 + .ci/scripts/install-tools.bat | 27 + .ci/scripts/kind-setup.sh | 18 + .editorconfig | 4 + .travis.yml | 3 + Jenkinsfile | 883 ++++++++++++++++-- Vagrantfile | 6 +- auditbeat/magefile.go | 27 +- dev-tools/mage/gotest.go | 9 +- dev-tools/mage/target/integtest/integtest.go | 7 +- filebeat/magefile.go | 25 +- filebeat/tests/system/test_tcp.py | 2 +- metricbeat/magefile.go | 26 +- .../system/diskio/diskstat_windows_test.go | 3 + script/fix_permissions.sh | 11 + winlogbeat/magefile.go | 1 + x-pack/auditbeat/magefile.go | 25 +- x-pack/dockerlogbeat/magefile.go | 7 +- x-pack/filebeat/magefile.go | 25 +- x-pack/functionbeat/magefile.go | 2 + x-pack/functionbeat/tests/system/test_base.py | 12 +- x-pack/libbeat/docker-compose.yml | 7 +- x-pack/libbeat/magefile.go | 15 +- x-pack/metricbeat/magefile.go | 26 +- x-pack/winlogbeat/magefile.go | 2 + 26 files changed, 994 insertions(+), 210 deletions(-) create mode 100755 .ci/scripts/install-docker-compose.sh create mode 100755 .ci/scripts/install-go.sh create mode 100755 .ci/scripts/install-tools.bat create mode 100755 .ci/scripts/kind-setup.sh create mode 100755 script/fix_permissions.sh diff --git a/.ci/scripts/install-docker-compose.sh b/.ci/scripts/install-docker-compose.sh new file mode 100755 index 00000000000..a430eb1873f --- /dev/null +++ b/.ci/scripts/install-docker-compose.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +MSG="parameter missing." +DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG} +HOME=${HOME:?$MSG} +DC_CMD="${HOME}/bin/docker-compose" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" +chmod +x "${DC_CMD}" diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh new file mode 100755 index 00000000000..5af9f338ca1 --- /dev/null +++ b/.ci/scripts/install-go.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="parameter missing." +GO_VERSION=${GO_VERSION:?$MSG} +PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} +HOME=${HOME:?$MSG} +ARCH=$(uname -s| tr '[:upper:]' '[:lower:]') +GVM_CMD="${HOME}/bin/gvm" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64" +chmod +x "${GVM_CMD}" + +gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} + +eval $(gvm ${GO_VERSION}) diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat new file mode 100755 index 00000000000..54bcaf5e8a4 --- /dev/null +++ b/.ci/scripts/install-tools.bat @@ -0,0 +1,27 @@ +set GOPATH=%WORKSPACE% +set MAGEFILE_CACHE=%WORKSPACE%\.magefile +set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH% + +where /q curl +IF ERRORLEVEL 1 ( + choco install curl -y --no-progress --skipdownloadcache +) +mkdir %WORKSPACE%\bin +where /q gvm +IF ERRORLEVEL 1 ( + curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe +) +FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i + +go env +go install -mod=vendor github.com/magefile/mage +mage -version +where mage + +if not exist C:\Python38\python.exe ( + REM Install python 3.8. + choco install python -y -r --no-progress --version 3.8.2 +) +python --version +where python + diff --git a/.ci/scripts/kind-setup.sh b/.ci/scripts/kind-setup.sh new file mode 100755 index 00000000000..4ac8fb0f6c3 --- /dev/null +++ b/.ci/scripts/kind-setup.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="parameter missing." +K8S_VERSION=${K8S_VERSION:?$MSG} +HOME=${HOME:?$MSG} +KBC_CMD="${HOME}/bin/kubectl" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" +chmod +x "${KBC_CMD}" + +GO111MODULE="on" go get sigs.k8s.io/kind@v0.5.1 +kind create cluster --image kindest/node:${K8S_VERSION} + +export KUBECONFIG="$(kind get kubeconfig-path)" +kubectl cluster-info diff --git a/.editorconfig b/.editorconfig index ff8d698c9a8..16a560fec2b 100644 --- a/.editorconfig +++ b/.editorconfig @@ -29,6 +29,10 @@ indent_style = tab [*.mk] indent_style = tab +[Jenkinsfile] +indent_size = 2 +indent_style = space + [Vagrantfile] indent_size = 2 indent_style = space diff --git a/.travis.yml b/.travis.yml index 67d601be003..e4f03562ebe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,9 @@ env: # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704 - TRAVIS_MINIKUBE_VERSION=v0.25.2 - MACOSX_DEPLOYMENT_TARGET=10.15 + # Enable verbose output since Travis users cannot access test output that is written to files. + # The constant output from tests also prevents time-outs. + - MAGEFILE_VERBOSE=true # Only run CI jobs on specific branches. # To keep the number of Travis CI jobs in check, we decided to configure Travis to only kick off CI jobs for the master branch. diff --git a/Jenkinsfile b/Jenkinsfile index d0887b757cd..a867093e503 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,125 +1,866 @@ #!/usr/bin/env groovy -library identifier: 'apm@current', -retriever: modernSCM( - [$class: 'GitSCMSource', - credentialsId: 'f94e9298-83ae-417e-ba91-85c279771570', - id: '37cf2c00-2cc7-482e-8c62-7bbffef475e2', - remote: 'git@github.com:elastic/apm-pipeline-library.git']) +@Library('apm@current') _ pipeline { - agent none + agent { label 'ubuntu && immutable' } environment { BASE_DIR = 'src/github.com/elastic/beats' + GOX_FLAGS = "-arch amd64" + DOCKER_COMPOSE_VERSION = "1.21.0" } options { - timeout(time: 1, unit: 'HOURS') + timeout(time: 2, unit: 'HOURS') buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) timestamps() ansiColor('xterm') disableResume() durabilityHint('PERFORMANCE_OPTIMIZED') + disableConcurrentBuilds() +// checkoutToSubdirectory "${env.BASE_DIR}" } triggers { issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') } parameters { - booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.') + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') + booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') } stages { /** Checkout the code and stash it, to use it on other stages. */ stage('Checkout') { - agent { label 'linux && immutable' } - environment { - PATH = "${env.PATH}:${env.WORKSPACE}/bin" - HOME = "${env.WORKSPACE}" - GOPATH = "${env.WORKSPACE}" - } options { skipDefaultCheckout() } steps { + deleteDir() + gitCheckout(basedir: "${BASE_DIR}") dir("${BASE_DIR}"){ - checkout scm + loadConfigEnvVars() } - stash allowEmpty: true, name: 'source', useDefaultExcludes: false - script { - env.GO_VERSION = readFile("${BASE_DIR}/.go-version") + whenTrue(params.debug){ + dumpFilteredEnvironment() } + stash allowEmpty: true, name: 'source', useDefaultExcludes: false } } - /** - Updating generated files for Beat. - Checks the GO environment. - Checks the Python environment. - Checks YAML files are generated. - Validate that all updates were committed. - */ - stage('Intake') { - agent { label 'linux && immutable' } + stage('Lint'){ options { skipDefaultCheckout() } - environment { - PATH = "${env.PATH}:${env.WORKSPACE}/bin" - HOME = "${env.WORKSPACE}" - GOPATH = "${env.WORKSPACE}" - } steps { - withGithubNotify(context: 'Intake') { - deleteDir() - unstash 'source' - dir("${BASE_DIR}"){ - sh './dev-tools/jenkins_intake.sh' - } - } + makeTarget("Lint", "check") } } - stage('Test') { - failFast true + stage('Build and Test'){ + failFast false parallel { - /** - Run unit tests and report junit results. - */ - stage('Filebeat') { - agent { label 'linux && immutable' } + stage('Filebeat oss'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" + } + } + steps { + makeTarget("Filebeat oss Linux", "-C filebeat testsuite") + } + } + stage('Filebeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT_XPACK != "false" + } + } + steps { + mageTarget("Filebeat x-pack Linux", "x-pack/filebeat", "update build test") + } + } + stage('Filebeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" && params.macosTest + } + } + steps { + makeTarget("Filebeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C filebeat testsuite") + } + } + stage('Filebeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" && params.windowsTest + } + } + steps { + mageTargetWin("Filebeat oss Windows Unit test", "filebeat", "unitTest") + } + } + stage('Heartbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_HEARTBEAT != "false" + } + } + stages { + stage('Heartbeat oss'){ + steps { + makeTarget("Heartbeat oss Linux", "-C heartbeat testsuite") + } + } + stage('Heartbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Heartbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C heartbeat testsuite") + } + } + stage('Heartbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Heartbeat oss Windows Unit test", "heartbeat", "unitTest") + } + } + } + } + stage('Auditbeat oss'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT != "false" + } + } + stages { + stage('Auditbeat Linux'){ + steps { + makeTarget("Auditbeat oss Linux", "-C auditbeat testsuite") + } + } + stage('Auditbeat crosscompile'){ + steps { + makeTarget("Auditbeat oss crosscompile", "-C auditbeat crosscompile") + } + } + stage('Auditbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Auditbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C auditbeat testsuite") + } + } + stage('Auditbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Auditbeat Windows Unit test", "auditbeat", "unitTest") + } + } + } + } + stage('Auditbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT_XPACK != "false" + } + } + steps { + mageTarget("Auditbeat x-pack Linux", "x-pack/auditbeat", "update build test") + } + } + stage('Libbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_LIBBEAT != "false" + } + } + stages { + stage('Libbeat oss'){ + steps { + makeTarget("Libbeat oss Linux", "-C libbeat testsuite") + } + } + stage('Libbeat crosscompile'){ + steps { + makeTarget("Libbeat oss crosscompile", "-C libbeat crosscompile") + } + } + stage('Libbeat stress-tests'){ + steps { + makeTarget("Libbeat stress-tests", "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") + } + } + } + } + stage('Libbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_LIBBEAT_XPACK != "false" + } + } + steps { + makeTarget("Libbeat x-pack Linux", "-C x-pack/libbeat testsuite") + } + } + stage('Metricbeat Unit tests'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat Unit tests", "-C metricbeat unit-tests coverage-report") + } + } + stage('Metricbeat Integration tests'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat Integration tests", "-C metricbeat integration-tests-environment coverage-report") + } + } + stage('Metricbeat System tests'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat System tests", "-C metricbeat update system-tests-environment coverage-report") + } + } + stage('Metricbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT_XPACK != "false" + } + } + steps { + mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "update build test") + } + } + stage('Metricbeat crosscompile'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat oss crosscompile", "-C metricbeat crosscompile") + } + } + stage('Metricbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" && params.macosTest + } + } + steps { + makeTarget("Metricbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C metricbeat testsuite") + } + } + stage('Metricbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" && params.windowsTest + } + } + steps { + mageTargetWin("Metricbeat Windows Unit test", "metricbeat", "unitTest") + } + } + stage('Packetbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_PACKETBEAT != "false" + } + } + stages { + stage('Packetbeat oss'){ + steps { + makeTarget("Packetbeat oss Linux", "-C packetbeat testsuite") + } + } + } + } + stage('dockerlogbeat'){ + agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } - environment { - PATH = "${env.PATH}:${env.WORKSPACE}/bin" - HOME = "${env.WORKSPACE}" - GOPATH = "${env.WORKSPACE}" + when { + beforeAgent true + expression { + return env.BUILD_DOCKERLOGBEAT_XPACK != "false" + } + } + stages { + stage('Dockerlogbeat'){ + steps { + mageTarget("Elastic Docker Logging Driver Plugin unit tests", "x-pack/dockerlogbeat", "update build test") + } + } + } + } + stage('Winlogbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_WINLOGBEAT != "false" + } + } + stages { + stage('Winlogbeat oss'){ + steps { + makeTarget("Winlogbeat oss crosscompile", "-C winlogbeat crosscompile") + } + } + stage('Winlogbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Winlogbeat Windows Unit test", "winlogbeat", "unitTest") + } + } + } + } + stage('Winlogbeat Windows x-pack'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" + } } steps { - withGithubNotify(context: 'Test', tab: 'tests') { - deleteDir() - unstash 'source' - dir("${BASE_DIR}"){ - sh './filebeat/scripts/jenkins/unit-test.sh' + mageTargetWin("Winlogbeat Windows Unit test", "x-pack/winlogbeat", "unitTest") + } + } + stage('Functionbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FUNCTIONBEAT_XPACK != "false" + } + } + stages { + stage('Functionbeat x-pack'){ + steps { + mageTarget("Functionbeat x-pack Linux", "x-pack/functionbeat", "update build test") + withEnv(["GO_VERSION=1.13.1"]){ + makeTarget("Functionbeat x-pack Linux", "-C x-pack/functionbeat test-gcp-functions") + } + } + } + stage('Functionbeat Mac OS X x-pack'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + mageTarget("Functionbeat x-pack Mac OS X", "x-pack/functionbeat", "update build test") + } + } + stage('Functionbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Functionbeat Windows Unit test", "x-pack/functionbeat", "unitTest") + } + } + } + } + stage('Journalbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_JOURNALBEAT != "false" + } + } + stages { + stage('Journalbeat oss'){ + steps { + makeTarget("Journalbeat Linux", "-C journalbeat testsuite") } } } - post { - always { - junit(allowEmptyResults: true, - keepLongStdio: true, - testResults: "${BASE_DIR}/build/junit-*.xml") + } + stage('Generators'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_GENERATOR != "false" } } + stages { + stage('Generators Metricbeat Linux'){ + steps { + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test") + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package") + } + } + stage('Generators Beat Linux'){ + steps { + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test") + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package") + } + } + stage('Generators Metricbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") + } + } + stage('Generators Beat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") + } + } + } + } + stage('Kubernetes'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_KUBERNETES != "false" + } + } + steps { + k8sTest(["v1.16.2","v1.15.3","v1.14.6","v1.13.10","v1.12.10","v1.11.10"]) + } + } + stage('Docs'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { return env.BUILD_DOCS != "false" } + } + steps { + makeTarget("Docs", "docs") + } } } } } - post { - success { - echoColor(text: '[SUCCESS]', colorfg: 'green', colorbg: 'default') +} + +def makeTarget(String context, String target, boolean clean = true) { + withGithubNotify(context: "${context}") { + withBeatsEnv(true) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + sh(label: "Make ${target}", script: "make ${target}") + if (clean) { + sh(script: 'script/fix_permissions.sh ${HOME}') + } + } + } +} + +def mageTarget(String context, String directory, String target) { + withGithubNotify(context: "${context}") { + withBeatsEnv(true) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } + } + } +} + +def mageTargetWin(String context, String directory, String target) { + withGithubNotify(context: "${context}") { + withBeatsEnvWin() { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMageWin() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } + } + } +} + +def withBeatsEnv(boolean archive, Closure body) { + def os = goos() + def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + + withEnv([ + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", + "MAGEFILE_CACHE=${WORKSPACE}/.magefile", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + "PYTHON_ENV=${WORKSPACE}/python-env", + "TEST_TAGS=oracle", + "DOCKER_PULL=0", + ]) { + deleteDir() + unstash 'source' + dir("${env.BASE_DIR}") { + sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") + sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") + sh(label: "Install Mage", script: "make mage") + try { + body() + } finally { + if (archive) { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') + } + } + reportCoverage() + } } - aborted { - echoColor(text: '[ABORTED]', colorfg: 'magenta', colorbg: 'default') + } +} + +def withBeatsEnvWin(Closure body) { + final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' + def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + + withEnv([ + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}", + "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + ]){ + deleteDir() + unstash 'source' + dir("${env.BASE_DIR}"){ + bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") + try { + body() + } finally { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') + } + } } - failure { - echoColor(text: '[FAILURE]', colorfg: 'red', colorbg: 'default') - //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${NOTIFY_TO}", sendToIndividuals: false]) + } +} + +def goos(){ + def labels = env.NODE_LABELS + + if (labels.contains('linux')) { + return 'linux' + } else if (labels.contains('windows')) { + return 'windows' + } else if (labels.contains('darwin')) { + return 'darwin' + } + + throw new IllegalArgumentException("Unhandled OS name in NODE_LABELS: " + labels) +} + +def dumpMage(){ + echo "### MAGE DUMP ###" + sh(label: "Dump mage variables", script: "mage dumpVariables") + echo "### END MAGE DUMP ###" +} + +def dumpMageWin(){ + echo "### MAGE DUMP ###" + bat(label: "Dump mage variables", script: "mage dumpVariables") + echo "### END MAGE DUMP ###" +} + +def dumpFilteredEnvironment(){ + echo "### ENV DUMP ###" + echo "PATH: ${env.PATH}" + echo "HOME: ${env.HOME}" + echo "USERPROFILE: ${env.USERPROFILE}" + echo "BUILD_DIR: ${env.BUILD_DIR}" + echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" + echo "BEATS: ${env.BEATS}" + echo "PROJECTS: ${env.PROJECTS}" + echo "PROJECTS_ENV: ${env.PROJECTS_ENV}" + echo "PYTHON_ENV: ${env.PYTHON_ENV}" + echo "PYTHON_EXE: ${env.PYTHON_EXE}" + echo "PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE}" + echo "VENV_PARAMS: ${env.VENV_PARAMS}" + echo "FIND: ${env.FIND}" + echo "GOLINT: ${env.GOLINT}" + echo "GOLINT_REPO: ${env.GOLINT_REPO}" + echo "REVIEWDOG: ${env.REVIEWDOG}" + echo "REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS}" + echo "REVIEWDOG_REPO: ${env.REVIEWDOG_REPO}" + echo "XPACK_SUFFIX: ${env.XPACK_SUFFIX}" + echo "PKG_BUILD_DIR: ${env.PKG_BUILD_DIR}" + echo "PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR}" + echo "COVERAGE_TOOL: ${env.COVERAGE_TOOL}" + echo "COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO}" + echo "TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO}" + echo "NOW: ${env.NOW}" + echo "GOBUILD_FLAGS: ${env.GOBUILD_FLAGS}" + echo "GOIMPORTS: ${env.GOIMPORTS}" + echo "GOIMPORTS_REPO: ${env.GOIMPORTS_REPO}" + echo "GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX}" + echo "PROCESSES: ${env.PROCESSES}" + echo "TIMEOUT: ${env.TIMEOUT}" + echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" + echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}" + echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" + echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" + echo "STRESS_TESTS: ${env.STRESS_TESTS}" + echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" + echo "GOX_OS: ${env.GOX_OS}" + echo "GOX_OSARCH: ${env.GOX_OSARCH}" + echo "GOX_FLAGS: ${env.GOX_FLAGS}" + echo "TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT}" + echo "BEAT_VERSION: ${env.BEAT_VERSION}" + echo "COMMIT_ID: ${env.COMMIT_ID}" + echo "DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME}" + echo "DOCKER_COMPOSE: ${env.DOCKER_COMPOSE}" + echo "DOCKER_CACHE: ${env.DOCKER_CACHE}" + echo "GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP}" + echo "PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS}" + echo "### END ENV DUMP ###" +} + +def k8sTest(versions){ + versions.each{ v -> + stage("k8s ${v}"){ + withEnv(["K8S_VERSION=${v}"]){ + withGithubNotify(context: "K8s ${v}") { + withBeatsEnv(false) { + sh(label: "Install k8s", script: """ + eval "\$(gvm use ${GO_VERSION} --format=bash)" + .ci/scripts/kind-setup.sh + """) + sh(label: "Kubernetes Kind",script: "make KUBECONFIG=\"\$(kind get kubeconfig-path)\" -C deploy/kubernetes test") + sh(label: 'Delete cluster', script: 'kind delete cluster') + } + } + } } - unstable { - echoColor(text: '[UNSTABLE]', colorfg: 'yellow', colorbg: 'default') + } +} + +def reportCoverage(){ + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + retry(2){ + sh(label: 'Report to Codecov', script: ''' + curl -sSLo codecov https://codecov.io/bash + for i in auditbeat filebeat heartbeat libbeat metricbeat packetbeat winlogbeat journalbeat + do + FILE="${i}/build/coverage/full.cov" + if [ -f "${FILE}" ]; then + bash codecov -f "${FILE}" + fi + done + ''') } } } + +def isChanged(patterns){ + return ( + params.runAllStages + || isGitRegionMatch(patterns: patterns, comparator: 'regexp') + ) +} + +def loadConfigEnvVars(){ + env.BUILD_AUDITBEAT = isChanged(["^auditbeat/.*"]) + env.BUILD_AUDITBEAT_XPACK = isChanged([ + "^auditbeat/.*", + "^x-pack/auditbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_DOCKERLOGBEAT_XPACK = isChanged([ + "^x-pack/dockerlogbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_DOCS = isChanged( + patterns: ["^docs/.*"], + comparator: 'regexp' + ) + env.BUILD_FILEBEAT = isChanged(["^filebeat/.*"]) + env.BUILD_FILEBEAT_XPACK = isChanged([ + "^filebeat/.*", + "^x-pack/filebeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_FUNCTIONBEAT_XPACK = isChanged([ + "^x-pack/functionbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_GENERATOR = isChanged(["^generator/.*"]) + env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"]) + env.BUILD_HEARTBEAT_XPACK = isChanged([ + "^heartbeat/.*", + "^x-pack/heartbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_JOURNALBEAT = isChanged(["^journalbeat/.*"]) + env.BUILD_JOURNALBEAT_XPACK = isChanged([ + "^journalbeat/.*", + "^x-pack/journalbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/*"]) + env.BUILD_LIBBEAT = isChanged( + patterns: ["^libbeat/.*"], + comparator: 'regexp' + ) + env.BUILD_LIBBEAT_XPACK = isChanged([ + "^libbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_METRICBEAT = isChanged(["^metricbeat/.*"]) + env.BUILD_METRICBEAT_XPACK = isChanged([ + "^metricbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/metricbeat/.*", + ]) + env.BUILD_PACKETBEAT = isChanged(["^packetbeat/.*"]) + env.BUILD_PACKETBEAT_XPACK = isChanged([ + "^packetbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/packetbeat/.*", + ]) + env.BUILD_WINLOGBEAT = isChanged(["^winlogbeat/.*"]) + env.BUILD_WINLOGBEAT_XPACK = isChanged([ + "^winlogbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/winlogbeat/.*", + ]) + env.GO_VERSION = readFile(".go-version").trim() +} diff --git a/Vagrantfile b/Vagrantfile index eba786adda2..fdff82a57ea 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -44,7 +44,7 @@ if (-Not (Test-Path $gopath_beats)) { if (-Not (Get-Command "gvm" -ErrorAction SilentlyContinue)) { echo "Installing gvm to manage go version" [Net.ServicePointManager]::SecurityProtocol = "tls12" - Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe + Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe C:\\Windows\\System32\\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression go version @@ -83,7 +83,7 @@ choco feature disable -n=showDownloadProgress if (-Not (Get-Command "python" -ErrorAction SilentlyContinue)) { echo "Installing python 3" - choco install python -y -r --version 3.8.1.20200110 + choco install python -y -r --version 3.8.2 refreshenv $env:PATH = "$env:PATH;C:\\Python38;C:\\Python38\\Scripts" } @@ -119,7 +119,7 @@ def linuxGvmProvision(arch="amd64") return <