diff --git a/.ci/scripts/install-docker-compose.sh b/.ci/scripts/install-docker-compose.sh new file mode 100755 index 000000000000..a430eb1873f4 --- /dev/null +++ b/.ci/scripts/install-docker-compose.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -exuo pipefail + +MSG="parameter missing." +DOCKER_COMPOSE_VERSION=${DOCKER_COMPOSE_VERSION:?$MSG} +HOME=${HOME:?$MSG} +DC_CMD="${HOME}/bin/docker-compose" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${DC_CMD}" "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" +chmod +x "${DC_CMD}" diff --git a/.ci/scripts/install-go.sh b/.ci/scripts/install-go.sh new file mode 100755 index 000000000000..5af9f338ca14 --- /dev/null +++ b/.ci/scripts/install-go.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="parameter missing." +GO_VERSION=${GO_VERSION:?$MSG} +PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"} +HOME=${HOME:?$MSG} +ARCH=$(uname -s| tr '[:upper:]' '[:lower:]') +GVM_CMD="${HOME}/bin/gvm" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64" +chmod +x "${GVM_CMD}" + +gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE} + +eval $(gvm ${GO_VERSION}) diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat new file mode 100755 index 000000000000..54bcaf5e8a44 --- /dev/null +++ b/.ci/scripts/install-tools.bat @@ -0,0 +1,27 @@ +set GOPATH=%WORKSPACE% +set MAGEFILE_CACHE=%WORKSPACE%\.magefile +set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;%PATH% + +where /q curl +IF ERRORLEVEL 1 ( + choco install curl -y --no-progress --skipdownloadcache +) +mkdir %WORKSPACE%\bin +where /q gvm +IF ERRORLEVEL 1 ( + curl -sL -o %WORKSPACE%\bin\gvm.exe https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe +) +FOR /f "tokens=*" %%i IN ('"gvm.exe" use %GO_VERSION% --format=batch') DO %%i + +go env +go install -mod=vendor github.com/magefile/mage +mage -version +where mage + +if not exist C:\Python38\python.exe ( + REM Install python 3.8. + choco install python -y -r --no-progress --version 3.8.2 +) +python --version +where python + diff --git a/.ci/scripts/kind-setup.sh b/.ci/scripts/kind-setup.sh new file mode 100755 index 000000000000..4ac8fb0f6c35 --- /dev/null +++ b/.ci/scripts/kind-setup.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -exuo pipefail + +MSG="parameter missing." +K8S_VERSION=${K8S_VERSION:?$MSG} +HOME=${HOME:?$MSG} +KBC_CMD="${HOME}/bin/kubectl" + +mkdir -p "${HOME}/bin" + +curl -sSLo "${KBC_CMD}" "https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl" +chmod +x "${KBC_CMD}" + +GO111MODULE="on" go get sigs.k8s.io/kind@v0.5.1 +kind create cluster --image kindest/node:${K8S_VERSION} + +export KUBECONFIG="$(kind get kubeconfig-path)" +kubectl cluster-info diff --git a/.editorconfig b/.editorconfig index ff8d698c9a8a..16a560fec2b2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -29,6 +29,10 @@ indent_style = tab [*.mk] indent_style = tab +[Jenkinsfile] +indent_size = 2 +indent_style = space + [Vagrantfile] indent_size = 2 indent_style = space diff --git a/.travis.yml b/.travis.yml index 67d601be0031..e4f03562ebee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,9 @@ env: # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704 - TRAVIS_MINIKUBE_VERSION=v0.25.2 - MACOSX_DEPLOYMENT_TARGET=10.15 + # Enable verbose output since Travis users cannot access test output that is written to files. + # The constant output from tests also prevents time-outs. + - MAGEFILE_VERBOSE=true # Only run CI jobs on specific branches. # To keep the number of Travis CI jobs in check, we decided to configure Travis to only kick off CI jobs for the master branch. diff --git a/Jenkinsfile b/Jenkinsfile index d0887b757cd4..a867093e503b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,125 +1,866 @@ #!/usr/bin/env groovy -library identifier: 'apm@current', -retriever: modernSCM( - [$class: 'GitSCMSource', - credentialsId: 'f94e9298-83ae-417e-ba91-85c279771570', - id: '37cf2c00-2cc7-482e-8c62-7bbffef475e2', - remote: 'git@github.com:elastic/apm-pipeline-library.git']) +@Library('apm@current') _ pipeline { - agent none + agent { label 'ubuntu && immutable' } environment { BASE_DIR = 'src/github.com/elastic/beats' + GOX_FLAGS = "-arch amd64" + DOCKER_COMPOSE_VERSION = "1.21.0" } options { - timeout(time: 1, unit: 'HOURS') + timeout(time: 2, unit: 'HOURS') buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) timestamps() ansiColor('xterm') disableResume() durabilityHint('PERFORMANCE_OPTIMIZED') + disableConcurrentBuilds() +// checkoutToSubdirectory "${env.BASE_DIR}" } triggers { issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') } parameters { - booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.') + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') + booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') } stages { /** Checkout the code and stash it, to use it on other stages. */ stage('Checkout') { - agent { label 'linux && immutable' } - environment { - PATH = "${env.PATH}:${env.WORKSPACE}/bin" - HOME = "${env.WORKSPACE}" - GOPATH = "${env.WORKSPACE}" - } options { skipDefaultCheckout() } steps { + deleteDir() + gitCheckout(basedir: "${BASE_DIR}") dir("${BASE_DIR}"){ - checkout scm + loadConfigEnvVars() } - stash allowEmpty: true, name: 'source', useDefaultExcludes: false - script { - env.GO_VERSION = readFile("${BASE_DIR}/.go-version") + whenTrue(params.debug){ + dumpFilteredEnvironment() } + stash allowEmpty: true, name: 'source', useDefaultExcludes: false } } - /** - Updating generated files for Beat. - Checks the GO environment. - Checks the Python environment. - Checks YAML files are generated. - Validate that all updates were committed. - */ - stage('Intake') { - agent { label 'linux && immutable' } + stage('Lint'){ options { skipDefaultCheckout() } - environment { - PATH = "${env.PATH}:${env.WORKSPACE}/bin" - HOME = "${env.WORKSPACE}" - GOPATH = "${env.WORKSPACE}" - } steps { - withGithubNotify(context: 'Intake') { - deleteDir() - unstash 'source' - dir("${BASE_DIR}"){ - sh './dev-tools/jenkins_intake.sh' - } - } + makeTarget("Lint", "check") } } - stage('Test') { - failFast true + stage('Build and Test'){ + failFast false parallel { - /** - Run unit tests and report junit results. - */ - stage('Filebeat') { - agent { label 'linux && immutable' } + stage('Filebeat oss'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" + } + } + steps { + makeTarget("Filebeat oss Linux", "-C filebeat testsuite") + } + } + stage('Filebeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT_XPACK != "false" + } + } + steps { + mageTarget("Filebeat x-pack Linux", "x-pack/filebeat", "update build test") + } + } + stage('Filebeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" && params.macosTest + } + } + steps { + makeTarget("Filebeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C filebeat testsuite") + } + } + stage('Filebeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FILEBEAT != "false" && params.windowsTest + } + } + steps { + mageTargetWin("Filebeat oss Windows Unit test", "filebeat", "unitTest") + } + } + stage('Heartbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_HEARTBEAT != "false" + } + } + stages { + stage('Heartbeat oss'){ + steps { + makeTarget("Heartbeat oss Linux", "-C heartbeat testsuite") + } + } + stage('Heartbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Heartbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C heartbeat testsuite") + } + } + stage('Heartbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Heartbeat oss Windows Unit test", "heartbeat", "unitTest") + } + } + } + } + stage('Auditbeat oss'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT != "false" + } + } + stages { + stage('Auditbeat Linux'){ + steps { + makeTarget("Auditbeat oss Linux", "-C auditbeat testsuite") + } + } + stage('Auditbeat crosscompile'){ + steps { + makeTarget("Auditbeat oss crosscompile", "-C auditbeat crosscompile") + } + } + stage('Auditbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Auditbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C auditbeat testsuite") + } + } + stage('Auditbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Auditbeat Windows Unit test", "auditbeat", "unitTest") + } + } + } + } + stage('Auditbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_AUDITBEAT_XPACK != "false" + } + } + steps { + mageTarget("Auditbeat x-pack Linux", "x-pack/auditbeat", "update build test") + } + } + stage('Libbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_LIBBEAT != "false" + } + } + stages { + stage('Libbeat oss'){ + steps { + makeTarget("Libbeat oss Linux", "-C libbeat testsuite") + } + } + stage('Libbeat crosscompile'){ + steps { + makeTarget("Libbeat oss crosscompile", "-C libbeat crosscompile") + } + } + stage('Libbeat stress-tests'){ + steps { + makeTarget("Libbeat stress-tests", "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") + } + } + } + } + stage('Libbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_LIBBEAT_XPACK != "false" + } + } + steps { + makeTarget("Libbeat x-pack Linux", "-C x-pack/libbeat testsuite") + } + } + stage('Metricbeat Unit tests'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat Unit tests", "-C metricbeat unit-tests coverage-report") + } + } + stage('Metricbeat Integration tests'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat Integration tests", "-C metricbeat integration-tests-environment coverage-report") + } + } + stage('Metricbeat System tests'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat System tests", "-C metricbeat update system-tests-environment coverage-report") + } + } + stage('Metricbeat x-pack'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT_XPACK != "false" + } + } + steps { + mageTarget("Metricbeat x-pack Linux", "x-pack/metricbeat", "update build test") + } + } + stage('Metricbeat crosscompile'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" + } + } + steps { + makeTarget("Metricbeat oss crosscompile", "-C metricbeat crosscompile") + } + } + stage('Metricbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" && params.macosTest + } + } + steps { + makeTarget("Metricbeat oss Mac OS X", "TEST_ENVIRONMENT=0 -C metricbeat testsuite") + } + } + stage('Metricbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_METRICBEAT != "false" && params.windowsTest + } + } + steps { + mageTargetWin("Metricbeat Windows Unit test", "metricbeat", "unitTest") + } + } + stage('Packetbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_PACKETBEAT != "false" + } + } + stages { + stage('Packetbeat oss'){ + steps { + makeTarget("Packetbeat oss Linux", "-C packetbeat testsuite") + } + } + } + } + stage('dockerlogbeat'){ + agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } - environment { - PATH = "${env.PATH}:${env.WORKSPACE}/bin" - HOME = "${env.WORKSPACE}" - GOPATH = "${env.WORKSPACE}" + when { + beforeAgent true + expression { + return env.BUILD_DOCKERLOGBEAT_XPACK != "false" + } + } + stages { + stage('Dockerlogbeat'){ + steps { + mageTarget("Elastic Docker Logging Driver Plugin unit tests", "x-pack/dockerlogbeat", "update build test") + } + } + } + } + stage('Winlogbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_WINLOGBEAT != "false" + } + } + stages { + stage('Winlogbeat oss'){ + steps { + makeTarget("Winlogbeat oss crosscompile", "-C winlogbeat crosscompile") + } + } + stage('Winlogbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Winlogbeat Windows Unit test", "winlogbeat", "unitTest") + } + } + } + } + stage('Winlogbeat Windows x-pack'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" + } } steps { - withGithubNotify(context: 'Test', tab: 'tests') { - deleteDir() - unstash 'source' - dir("${BASE_DIR}"){ - sh './filebeat/scripts/jenkins/unit-test.sh' + mageTargetWin("Winlogbeat Windows Unit test", "x-pack/winlogbeat", "unitTest") + } + } + stage('Functionbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_FUNCTIONBEAT_XPACK != "false" + } + } + stages { + stage('Functionbeat x-pack'){ + steps { + mageTarget("Functionbeat x-pack Linux", "x-pack/functionbeat", "update build test") + withEnv(["GO_VERSION=1.13.1"]){ + makeTarget("Functionbeat x-pack Linux", "-C x-pack/functionbeat test-gcp-functions") + } + } + } + stage('Functionbeat Mac OS X x-pack'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + mageTarget("Functionbeat x-pack Mac OS X", "x-pack/functionbeat", "update build test") + } + } + stage('Functionbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } + steps { + mageTargetWin("Functionbeat Windows Unit test", "x-pack/functionbeat", "unitTest") + } + } + } + } + stage('Journalbeat'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_JOURNALBEAT != "false" + } + } + stages { + stage('Journalbeat oss'){ + steps { + makeTarget("Journalbeat Linux", "-C journalbeat testsuite") } } } - post { - always { - junit(allowEmptyResults: true, - keepLongStdio: true, - testResults: "${BASE_DIR}/build/junit-*.xml") + } + stage('Generators'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_GENERATOR != "false" } } + stages { + stage('Generators Metricbeat Linux'){ + steps { + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test") + makeTarget("Generators Metricbeat Linux", "-C generator/_templates/metricbeat test-package") + } + } + stage('Generators Beat Linux'){ + steps { + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test") + makeTarget("Generators Beat Linux", "-C generator/_templates/beat test-package") + } + } + stage('Generators Metricbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Generators Metricbeat Mac OS X", "-C generator/_templates/metricbeat test") + } + } + stage('Generators Beat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + makeTarget("Generators Beat Mac OS X", "-C generator/_templates/beat test") + } + } + } + } + stage('Kubernetes'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return env.BUILD_KUBERNETES != "false" + } + } + steps { + k8sTest(["v1.16.2","v1.15.3","v1.14.6","v1.13.10","v1.12.10","v1.11.10"]) + } + } + stage('Docs'){ + agent { label 'ubuntu && immutable' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { return env.BUILD_DOCS != "false" } + } + steps { + makeTarget("Docs", "docs") + } } } } } - post { - success { - echoColor(text: '[SUCCESS]', colorfg: 'green', colorbg: 'default') +} + +def makeTarget(String context, String target, boolean clean = true) { + withGithubNotify(context: "${context}") { + withBeatsEnv(true) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + sh(label: "Make ${target}", script: "make ${target}") + if (clean) { + sh(script: 'script/fix_permissions.sh ${HOME}') + } + } + } +} + +def mageTarget(String context, String directory, String target) { + withGithubNotify(context: "${context}") { + withBeatsEnv(true) { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMage() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } + } + } +} + +def mageTargetWin(String context, String directory, String target) { + withGithubNotify(context: "${context}") { + withBeatsEnvWin() { + whenTrue(params.debug) { + dumpFilteredEnvironment() + dumpMageWin() + } + + def verboseFlag = params.debug ? "-v" : "" + dir(directory) { + bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + } + } + } +} + +def withBeatsEnv(boolean archive, Closure body) { + def os = goos() + def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" + + withEnv([ + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", + "MAGEFILE_CACHE=${WORKSPACE}/.magefile", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + "PYTHON_ENV=${WORKSPACE}/python-env", + "TEST_TAGS=oracle", + "DOCKER_PULL=0", + ]) { + deleteDir() + unstash 'source' + dir("${env.BASE_DIR}") { + sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") + sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") + sh(label: "Install Mage", script: "make mage") + try { + body() + } finally { + if (archive) { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') + } + } + reportCoverage() + } } - aborted { - echoColor(text: '[ABORTED]', colorfg: 'magenta', colorbg: 'default') + } +} + +def withBeatsEnvWin(Closure body) { + final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' + def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + + withEnv([ + "HOME=${env.WORKSPACE}", + "GOPATH=${env.WORKSPACE}", + "GOROOT=${goRoot}", + "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}", + "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", + "TEST_COVERAGE=true", + "RACE_DETECTOR=true", + ]){ + deleteDir() + unstash 'source' + dir("${env.BASE_DIR}"){ + bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") + try { + body() + } finally { + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") + archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') + } + } } - failure { - echoColor(text: '[FAILURE]', colorfg: 'red', colorbg: 'default') - //step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${NOTIFY_TO}", sendToIndividuals: false]) + } +} + +def goos(){ + def labels = env.NODE_LABELS + + if (labels.contains('linux')) { + return 'linux' + } else if (labels.contains('windows')) { + return 'windows' + } else if (labels.contains('darwin')) { + return 'darwin' + } + + throw new IllegalArgumentException("Unhandled OS name in NODE_LABELS: " + labels) +} + +def dumpMage(){ + echo "### MAGE DUMP ###" + sh(label: "Dump mage variables", script: "mage dumpVariables") + echo "### END MAGE DUMP ###" +} + +def dumpMageWin(){ + echo "### MAGE DUMP ###" + bat(label: "Dump mage variables", script: "mage dumpVariables") + echo "### END MAGE DUMP ###" +} + +def dumpFilteredEnvironment(){ + echo "### ENV DUMP ###" + echo "PATH: ${env.PATH}" + echo "HOME: ${env.HOME}" + echo "USERPROFILE: ${env.USERPROFILE}" + echo "BUILD_DIR: ${env.BUILD_DIR}" + echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" + echo "BEATS: ${env.BEATS}" + echo "PROJECTS: ${env.PROJECTS}" + echo "PROJECTS_ENV: ${env.PROJECTS_ENV}" + echo "PYTHON_ENV: ${env.PYTHON_ENV}" + echo "PYTHON_EXE: ${env.PYTHON_EXE}" + echo "PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE}" + echo "VENV_PARAMS: ${env.VENV_PARAMS}" + echo "FIND: ${env.FIND}" + echo "GOLINT: ${env.GOLINT}" + echo "GOLINT_REPO: ${env.GOLINT_REPO}" + echo "REVIEWDOG: ${env.REVIEWDOG}" + echo "REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS}" + echo "REVIEWDOG_REPO: ${env.REVIEWDOG_REPO}" + echo "XPACK_SUFFIX: ${env.XPACK_SUFFIX}" + echo "PKG_BUILD_DIR: ${env.PKG_BUILD_DIR}" + echo "PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR}" + echo "COVERAGE_TOOL: ${env.COVERAGE_TOOL}" + echo "COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO}" + echo "TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO}" + echo "NOW: ${env.NOW}" + echo "GOBUILD_FLAGS: ${env.GOBUILD_FLAGS}" + echo "GOIMPORTS: ${env.GOIMPORTS}" + echo "GOIMPORTS_REPO: ${env.GOIMPORTS_REPO}" + echo "GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX}" + echo "PROCESSES: ${env.PROCESSES}" + echo "TIMEOUT: ${env.TIMEOUT}" + echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" + echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}" + echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" + echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" + echo "STRESS_TESTS: ${env.STRESS_TESTS}" + echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" + echo "GOX_OS: ${env.GOX_OS}" + echo "GOX_OSARCH: ${env.GOX_OSARCH}" + echo "GOX_FLAGS: ${env.GOX_FLAGS}" + echo "TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT}" + echo "BEAT_VERSION: ${env.BEAT_VERSION}" + echo "COMMIT_ID: ${env.COMMIT_ID}" + echo "DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME}" + echo "DOCKER_COMPOSE: ${env.DOCKER_COMPOSE}" + echo "DOCKER_CACHE: ${env.DOCKER_CACHE}" + echo "GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP}" + echo "PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS}" + echo "### END ENV DUMP ###" +} + +def k8sTest(versions){ + versions.each{ v -> + stage("k8s ${v}"){ + withEnv(["K8S_VERSION=${v}"]){ + withGithubNotify(context: "K8s ${v}") { + withBeatsEnv(false) { + sh(label: "Install k8s", script: """ + eval "\$(gvm use ${GO_VERSION} --format=bash)" + .ci/scripts/kind-setup.sh + """) + sh(label: "Kubernetes Kind",script: "make KUBECONFIG=\"\$(kind get kubeconfig-path)\" -C deploy/kubernetes test") + sh(label: 'Delete cluster', script: 'kind delete cluster') + } + } + } } - unstable { - echoColor(text: '[UNSTABLE]', colorfg: 'yellow', colorbg: 'default') + } +} + +def reportCoverage(){ + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { + retry(2){ + sh(label: 'Report to Codecov', script: ''' + curl -sSLo codecov https://codecov.io/bash + for i in auditbeat filebeat heartbeat libbeat metricbeat packetbeat winlogbeat journalbeat + do + FILE="${i}/build/coverage/full.cov" + if [ -f "${FILE}" ]; then + bash codecov -f "${FILE}" + fi + done + ''') } } } + +def isChanged(patterns){ + return ( + params.runAllStages + || isGitRegionMatch(patterns: patterns, comparator: 'regexp') + ) +} + +def loadConfigEnvVars(){ + env.BUILD_AUDITBEAT = isChanged(["^auditbeat/.*"]) + env.BUILD_AUDITBEAT_XPACK = isChanged([ + "^auditbeat/.*", + "^x-pack/auditbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_DOCKERLOGBEAT_XPACK = isChanged([ + "^x-pack/dockerlogbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_DOCS = isChanged( + patterns: ["^docs/.*"], + comparator: 'regexp' + ) + env.BUILD_FILEBEAT = isChanged(["^filebeat/.*"]) + env.BUILD_FILEBEAT_XPACK = isChanged([ + "^filebeat/.*", + "^x-pack/filebeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_FUNCTIONBEAT_XPACK = isChanged([ + "^x-pack/functionbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_GENERATOR = isChanged(["^generator/.*"]) + env.BUILD_HEARTBEAT = isChanged(["^heartbeat/.*"]) + env.BUILD_HEARTBEAT_XPACK = isChanged([ + "^heartbeat/.*", + "^x-pack/heartbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_JOURNALBEAT = isChanged(["^journalbeat/.*"]) + env.BUILD_JOURNALBEAT_XPACK = isChanged([ + "^journalbeat/.*", + "^x-pack/journalbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/*"]) + env.BUILD_LIBBEAT = isChanged( + patterns: ["^libbeat/.*"], + comparator: 'regexp' + ) + env.BUILD_LIBBEAT_XPACK = isChanged([ + "^libbeat/.*", + "^x-pack/libbeat/.*", + ]) + env.BUILD_METRICBEAT = isChanged(["^metricbeat/.*"]) + env.BUILD_METRICBEAT_XPACK = isChanged([ + "^metricbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/metricbeat/.*", + ]) + env.BUILD_PACKETBEAT = isChanged(["^packetbeat/.*"]) + env.BUILD_PACKETBEAT_XPACK = isChanged([ + "^packetbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/packetbeat/.*", + ]) + env.BUILD_WINLOGBEAT = isChanged(["^winlogbeat/.*"]) + env.BUILD_WINLOGBEAT_XPACK = isChanged([ + "^winlogbeat/.*", + "^x-pack/libbeat/.*", + "^x-pack/winlogbeat/.*", + ]) + env.GO_VERSION = readFile(".go-version").trim() +} diff --git a/Vagrantfile b/Vagrantfile index eba786adda20..fdff82a57ea1 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -44,7 +44,7 @@ if (-Not (Test-Path $gopath_beats)) { if (-Not (Get-Command "gvm" -ErrorAction SilentlyContinue)) { echo "Installing gvm to manage go version" [Net.ServicePointManager]::SecurityProtocol = "tls12" - Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe + Invoke-WebRequest -URI https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-windows-amd64.exe -Outfile C:\\Windows\\System32\\gvm.exe C:\\Windows\\System32\\gvm.exe --format=powershell #{GO_VERSION} | Invoke-Expression go version @@ -83,7 +83,7 @@ choco feature disable -n=showDownloadProgress if (-Not (Get-Command "python" -ErrorAction SilentlyContinue)) { echo "Installing python 3" - choco install python -y -r --version 3.8.1.20200110 + choco install python -y -r --version 3.8.2 refreshenv $env:PATH = "$env:PATH;C:\\Python38;C:\\Python38\\Scripts" } @@ -119,7 +119,7 @@ def linuxGvmProvision(arch="amd64") return <