From e076e9a1ccb90aaef2b11d1ebc161437cf40097a Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 28 Sep 2020 09:06:21 +0100 Subject: [PATCH] [CI] Pipeline 2.0 for monorepos (#20104) (#21312) --- .ci/scripts/generate_build_table.py | 52 + .ci/scripts/get-vendor-dependencies.sh | 16 + .ci/scripts/install-tools.sh | 7 + Jenkinsfile | 1591 ++++++------------------ Jenkinsfile.yml | 47 + README.md | 12 +- auditbeat/Jenkinsfile.yml | 34 + deploy/kubernetes/Jenkinsfile.yml | 15 + filebeat/Jenkinsfile.yml | 33 + generator/Jenkinsfile.yml | 43 + heartbeat/Jenkinsfile.yml | 32 + journalbeat/Jenkinsfile.yml | 17 + libbeat/Jenkinsfile.yml | 20 + metricbeat/Jenkinsfile.yml | 40 + packetbeat/Jenkinsfile.yml | 32 + winlogbeat/Jenkinsfile.yml | 21 + x-pack/auditbeat/Jenkinsfile.yml | 33 + x-pack/dockerlogbeat/Jenkinsfile.yml | 18 + x-pack/elastic-agent/Jenkinsfile.yml | 32 + x-pack/filebeat/Jenkinsfile.yml | 33 + x-pack/functionbeat/Jenkinsfile.yml | 32 + x-pack/libbeat/Jenkinsfile.yml | 17 + x-pack/metricbeat/Jenkinsfile.yml | 35 + x-pack/packetbeat/Jenkinsfile.yml | 20 + x-pack/winlogbeat/Jenkinsfile.yml | 20 + 25 files changed, 1070 insertions(+), 1182 deletions(-) create mode 100755 .ci/scripts/generate_build_table.py create mode 100755 .ci/scripts/get-vendor-dependencies.sh create mode 100755 .ci/scripts/install-tools.sh create mode 100644 Jenkinsfile.yml create mode 100644 auditbeat/Jenkinsfile.yml create mode 100644 deploy/kubernetes/Jenkinsfile.yml create mode 100644 filebeat/Jenkinsfile.yml create mode 100644 generator/Jenkinsfile.yml create mode 100644 heartbeat/Jenkinsfile.yml create mode 100644 journalbeat/Jenkinsfile.yml create mode 100644 libbeat/Jenkinsfile.yml create mode 100644 metricbeat/Jenkinsfile.yml create mode 100644 packetbeat/Jenkinsfile.yml create mode 100644 winlogbeat/Jenkinsfile.yml create mode 100644 x-pack/auditbeat/Jenkinsfile.yml create mode 100644 x-pack/dockerlogbeat/Jenkinsfile.yml create mode 100644 x-pack/elastic-agent/Jenkinsfile.yml create mode 100644 x-pack/filebeat/Jenkinsfile.yml create mode 100644 x-pack/functionbeat/Jenkinsfile.yml create mode 100644 x-pack/libbeat/Jenkinsfile.yml create mode 100644 x-pack/metricbeat/Jenkinsfile.yml create mode 100644 x-pack/packetbeat/Jenkinsfile.yml create mode 100644 x-pack/winlogbeat/Jenkinsfile.yml diff --git a/.ci/scripts/generate_build_table.py b/.ci/scripts/generate_build_table.py new file mode 100755 index 00000000000..21fe5d67107 --- /dev/null +++ b/.ci/scripts/generate_build_table.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 + +import os +import yaml + +if __name__ == "__main__": + + print("| Beat | Stage | Command | MODULE | Platforms | When |") + print("|-------|--------|----------|---------|------------|------|") + for root, dirs, files in os.walk("."): + dirs.sort() + for file in files: + if file.endswith("Jenkinsfile.yml") and root != ".": + with open(os.path.join(root, file), 'r') as f: + doc = yaml.load(f, Loader=yaml.FullLoader) + module = root.replace(".{}".format(os.sep), '') + platforms = [doc["platform"]] + when = "" + if "branches" in doc["when"]: + when = f"{when}/:palm_tree:" + if "changeset" in doc["when"]: + when = f"{when}/:file_folder:" + if "comments" in doc["when"]: + when = f"{when}/:speech_balloon:" + if "labels" in doc["when"]: + when = f"{when}/:label:" + if "parameters" in doc["when"]: + when = f"{when}/:smiley:" + if "tags" in doc["when"]: + when = f"{when}/:taco:" + for stage in doc["stages"]: + withModule = False + if "make" in doc["stages"][stage]: + command = doc["stages"][stage]["make"] + if "mage" in doc["stages"][stage]: + command = doc["stages"][stage]["mage"] + if "platforms" in doc["stages"][stage]: + platforms = doc["stages"][stage]["platforms"] + if "withModule" in doc["stages"][stage]: + withModule = doc["stages"][stage]["withModule"] + if "when" in doc["stages"][stage]: + when = f"{when}/:star:" + print("| {} | {} | `{}` | {} | `{}` | {} |".format( + module, stage, command, withModule, platforms, when)) + +print("> :palm_tree: -> Git Branch based") +print("> :label: -> GitHub Pull Request Label based") +print("> :file_folder: -> Changeset based") +print("> :speech_balloon: -> GitHub Pull Request comment based") +print("> :taco: -> Git tag based") +print("> :smiley: -> Manual UI interaction based") +print("> :star: -> More specific cases based") diff --git a/.ci/scripts/get-vendor-dependencies.sh b/.ci/scripts/get-vendor-dependencies.sh new file mode 100755 index 00000000000..e002a208b76 --- /dev/null +++ b/.ci/scripts/get-vendor-dependencies.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Given the go module it will list all the dependencies that will be later on +# used by the CI to enable/disable specific stages as long as the changeset +# matches any of those patterns. +# + +GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"} +BEATS=${1:?"parameter missing."} +eval "$(gvm "${GO_VERSION}")" + +go list -deps ./"${BEATS}" \ +| grep 'elastic/beats' \ +| sort \ +| sed -e "s#github.com/elastic/beats/v7/##g" \ +| awk '{print "^" $1 "/.*"}' diff --git a/.ci/scripts/install-tools.sh b/.ci/scripts/install-tools.sh new file mode 100755 index 00000000000..297a7820cad --- /dev/null +++ b/.ci/scripts/install-tools.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -exuo pipefail + +.ci/scripts/install-go.sh +.ci/scripts/install-docker-compose.sh +.ci/scripts/install-terraform.sh +make mage diff --git a/Jenkinsfile b/Jenkinsfile index c3b79ca98ca..335e034e746 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,19 +12,19 @@ import groovy.transform.Field pipeline { agent { label 'ubuntu-18 && immutable' } environment { + AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' BASE_DIR = 'src/github.com/elastic/beats' - GOX_FLAGS = "-arch amd64" - DOCKER_COMPOSE_VERSION = "1.21.0" - TERRAFORM_VERSION = "0.12.24" - PIPELINE_LOG_LEVEL = "INFO" DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + DOCKER_COMPOSE_VERSION = "1.21.0" DOCKER_REGISTRY = 'docker.elastic.co' - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' - RUNBLD_DISABLE_NOTIFICATIONS = 'true' + GOX_FLAGS = "-arch amd64" JOB_GCS_BUCKET = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' + PIPELINE_LOG_LEVEL = 'INFO' + RUNBLD_DISABLE_NOTIFICATIONS = 'true' + TERRAFORM_VERSION = "0.12.24" + XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*' } options { timeout(time: 2, unit: 'HOURS') @@ -37,22 +37,16 @@ pipeline { rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) } triggers { - issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test(\\W+macos)?$)') + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*|^/test\\W+.*$)') } parameters { - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') - booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') - booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') + booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.') string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') - booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') - booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow') + booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') + booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') } stages { - /** - Checkout the code and stash it, to use it on other stages. - */ stage('Checkout') { options { skipDefaultCheckout() } steps { @@ -61,711 +55,65 @@ pipeline { gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") dir("${BASE_DIR}"){ - loadConfigEnvVars() - } - whenTrue(params.debug){ - dumpFilteredEnvironment() + // Skip all the stages except docs for PR's with asciidoc and md changes only + setEnvVar('ONLY_DOCS', isGitRegionMatch(patterns: [ '.*\\.(asciidoc|md)' ], shouldMatchAll: true).toString()) + setEnvVar('GO_VERSION', readFile(".go-version").trim()) + withEnv(["HOME=${env.WORKSPACE}"]) { + retryWithSleep(retries: 2, seconds: 5){ sh(label: "Install Go ${env.GO_VERSION}", script: '.ci/scripts/install-go.sh') } + } } } } stage('Lint'){ options { skipDefaultCheckout() } environment { - // See https://github.com/elastic/beats/pull/19823 GOFLAGS = '-mod=readonly' } steps { - makeTarget(context: "Lint", target: "check") + withGithubNotify(context: 'Lint') { + withBeatsEnv(archive: true) { + dumpVariables() + cmd(label: 'make check', script: 'make check') + } + } } } - stage('Build and Test'){ + stage('Build&Test') { + options { skipDefaultCheckout() } when { - beforeAgent true - expression { return env.ONLY_DOCS == "false" } - } - failFast false - parallel { - stage('Elastic Agent x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" - } - } - steps { - mageTarget(context: "Elastic Agent x-pack Linux", directory: "x-pack/elastic-agent", target: "build test") - } - } - stage('Elastic Agent x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Elastic Agent x-pack Windows Unit test", directory: "x-pack/elastic-agent", target: "build unitTest") - } - } - stage('Elastic Agent Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ELASTIC_AGENT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Elastic Agent x-pack Mac OS X", directory: "x-pack/elastic-agent", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat oss'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" - } - } - steps { - mageTarget(context: "Filebeat oss Linux", directory: "filebeat", target: "build test", withModule: true) - } - } - stage('Filebeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "build test", withModule: true) - } - } - stage('Filebeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Filebeat oss Mac OS X", directory: "filebeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Filebeat x-pack Mac OS X", directory: "x-pack/filebeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Filebeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Filebeat oss Windows Unit test", directory: "filebeat", target: "build unitTest") - } - } - stage('Filebeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FILEBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Filebeat x-pack Windows", directory: "x-pack/filebeat", target: "build unitTest") - } - } - stage('Heartbeat oss'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_HEARTBEAT != "false" - } - } - steps { - mageTarget(context: "Heartbeat oss Linux", directory: "heartbeat", target: "build test") - } - } - stage('Heartbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_HEARTBEAT != "false" - } - } - steps { - mageTarget(context: "Heartbeat oss Mac OS X", directory: "heartbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Heartbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_HEARTBEAT != "false" - } - } - steps { - mageTargetWin(context: "Heartbeat oss Windows Unit test", directory: "heartbeat", target: "build unitTest") - } - } - stage('Auditbeat oss Linux'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" - } - } - steps { - mageTarget(context: "Auditbeat oss Linux", directory: "auditbeat", target: "build test") - } - } - stage('Auditbeat crosscompile'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" - } - } - steps { - makeTarget(context: "Auditbeat oss crosscompile", directory: 'auditbeat', target: "crosscompile") - } - } - stage('Auditbeat oss Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Auditbeat oss Mac OS X", directory: "auditbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Auditbeat oss Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Auditbeat oss Windows Unit test", directory: "auditbeat", target: "build unitTest") - } - } - stage('Auditbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Auditbeat x-pack Linux", directory: "x-pack/auditbeat", target: "update build test", withModule: true) - } - } - stage('Auditbeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Auditbeat x-pack Mac OS X", directory: "x-pack/auditbeat", target: "build unitTest") - } - } - stage('Auditbeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_AUDITBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Auditbeat x-pack Windows", directory: "x-pack/auditbeat", target: "build unitTest") - } - } - stage('Libbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_LIBBEAT != "false" - } - } - stages { - stage('Libbeat oss'){ - steps { - mageTarget(context: "Libbeat oss Linux", directory: "libbeat", target: "build test") - } - } - stage('Libbeat crosscompile'){ - steps { - makeTarget(context: "Libbeat oss crosscompile", directory: 'libbeat', target: "crosscompile") - } - } - stage('Libbeat stress-tests'){ - steps { - makeTarget(context: "Libbeat stress-tests", target: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") - } - } - } - } - stage('Libbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_LIBBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Libbeat x-pack Linux", directory: "x-pack/libbeat", target: "build test") - } - } - stage('Metricbeat OSS Unit tests'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (unitTest)", directory: "metricbeat", target: "build unitTest") + // Always when running builds on branches/tags + // On a PR basis, skip if changes are only related to docs. + // Always when forcing the input parameter + anyOf { + not { changeRequest() } // If no PR + allOf { // If PR and no docs changes + expression { return env.ONLY_DOCS == "false" } + changeRequest() } + expression { return params.runAllStages } // If UI forced } - stage('Metricbeat OSS Go Integration tests'){ - agent { label 'ubuntu && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (goIntegTest)", directory: "metricbeat", target: "goIntegTest", withModule: true) - } - } - stage('Metricbeat OSS Python Integration tests'){ - agent { label 'ubuntu && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - mageTarget(context: "Metricbeat OSS linux/amd64 (pythonIntegTest)", directory: "metricbeat", target: "pythonIntegTest", withModule: true) - } - } - stage('Metricbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" - } - } - stages { - stage('Prepare cloud integration tests environments'){ - options { skipDefaultCheckout() } - steps { - startCloudTestEnv('x-pack-metricbeat', [ - [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], - ]) - } - } - stage('Metricbeat x-pack'){ - options { skipDefaultCheckout() } - steps { - withCloudTestEnv() { - mageTarget(context: "Metricbeat x-pack Linux", directory: "x-pack/metricbeat", target: "build test", withModule: true) - } - } - } - } - post { - cleanup { - terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') - } - } - } - stage('Metricbeat crosscompile'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" - } - } - steps { - makeTarget(context: "Metricbeat OSS crosscompile", directory: 'metricbeat', target: "crosscompile") - } - } - stage('Metricbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Metricbeat OSS Mac OS X", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat x-pack Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" && env.BUILD_ON_MACOS != 'false' - } - } - steps { - mageTarget(context: "Metricbeat x-pack Mac OS X", directory: "x-pack/metricbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Metricbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Metricbeat Windows Unit test", directory: "metricbeat", target: "build unitTest") - } - } - stage('Metricbeat x-pack Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_METRICBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin(context: "Metricbeat x-pack Windows", directory: "x-pack/metricbeat", target: "build unitTest") - } - } - stage('Packetbeat Linux'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_PACKETBEAT != "false" - } - } - steps { - mageTarget(context: "Packetbeat OSS Linux", directory: "packetbeat", target: "build test") - } - } - stage('Packetbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_PACKETBEAT != "false" - } - } - steps { - mageTarget(context: "Packetbeat OSS Mac OS X", directory: "packetbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Packetbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_PACKETBEAT != "false" - } - } - steps { - mageTargetWin(context: "Packetbeat OSS Windows", directory: "packetbeat", target: "build unitTest") - } - } - stage('dockerlogbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_DOCKERLOGBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "build test") - } - } - stage('Winlogbeat oss'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_WINLOGBEAT != "false" - } - } - steps { - makeTarget(context: "Winlogbeat oss crosscompile", directory: 'winlogbeat', target: "crosscompile") - } - } - stage('Winlogbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_WINLOGBEAT != "false" - } - } - steps { - mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "winlogbeat", target: "build unitTest") - } - } - stage('Winlogbeat Windows x-pack'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" - } - } - steps { - mageTargetWin(context: "Winlogbeat Windows Unit test", directory: "x-pack/winlogbeat", target: "build unitTest", withModule: true) - } - } - stage('Functionbeat x-pack'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") - withEnv(["GO_VERSION=1.13.1"]){ - mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "testGCPFunctions") - } - } - } - stage('Functionbeat Mac OS X x-pack'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - steps { - mageTarget(context: "Functionbeat x-pack Mac OS X", directory: "x-pack/functionbeat", target: "build unitTest") - } - post { - always { - delete() - } - } - } - stage('Functionbeat Windows'){ - agent { label 'windows-immutable && windows-2019' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest && env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - steps { - mageTargetWin(context: "Functionbeat Windows Unit test", directory: "x-pack/functionbeat", target: "build unitTest") - } - } - stage('Journalbeat'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_JOURNALBEAT != "false" - } - } - steps { - mageTarget(context: "Journalbeat Linux", directory: "journalbeat", target: "build unitTest") - } - } - stage('Generators'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_GENERATOR != "false" - } - } - stages { - stage('Generators Metricbeat Linux'){ - steps { - makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test") - makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test-package") - } - } - stage('Generators Beat Linux'){ - steps { - makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test") - makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test-package") + } + steps { + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${BASE_DIR}"){ + script { + def mapParallelTasks = [:] + def content = readYaml(file: 'Jenkinsfile.yml') + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset']).each { k,v -> + mapParallelTasks["${k}"] = v } } + parallel(mapParallelTasks) } } - stage('Generators Metricbeat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_GENERATOR != "false" - } - } - steps { - makeTarget(context: "Generators Metricbeat Mac OS X", directory: 'generator/_templates/metricbeat', target: "test") - } - post { - always { - delete() - } - } - } - stage('Generators Beat Mac OS X'){ - agent { label 'macosx' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_ON_MACOS != 'false' && env.BUILD_GENERATOR != "false" - } - } - steps { - makeTarget(context: "Generators Beat Mac OS X", directory: 'generator/_templates/beat', target: "test") - } - post { - always { - delete() - } - } - } - stage('Kubernetes'){ - agent { label 'ubuntu-18 && immutable' } - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return env.BUILD_KUBERNETES != "false" - } - } - steps { - k8sTest(["v1.18.2","v1.17.2","v1.16.4","v1.15.7","v1.14.10"]) + } + post { + always { + dir("${BASE_DIR}"){ + // Archive the markdown files that contain the build reasons + archiveArtifacts(allowEmptyArchive: false, artifacts: 'build-reasons/*.md') } } } @@ -781,131 +129,158 @@ pipeline { } } -def delete() { - dir("${env.BASE_DIR}") { - fixPermissions("${WORKSPACE}") +/** +* This method is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +*/ +def generateStages(Map args = [:]) { + def projectName = args.project + def changeset = args.changeset + def mapParallelStages = [:] + def fileName = "${projectName}/Jenkinsfile.yml" + if (fileExists(fileName)) { + def content = readYaml(file: fileName) + // changesetFunction argument is only required for the top-level when, stage specific when don't need it since it's an aggregation. + if (beatsWhen(project: projectName, content: content?.when, changeset: changeset, changesetFunction: new GetProjectDependencies(steps: this))) { + mapParallelStages = beatsStages(project: projectName, content: content, changeset: changeset, function: new RunCommand(steps: this)) + } + } else { + log(level: 'WARN', text: "${fileName} file does not exist. Please review the top-level Jenkinsfile.yml") } - deleteDir() -} - -def fixPermissions(location) { - sh(label: 'Fix permissions', script: """#!/usr/bin/env bash - source ./dev-tools/common.bash - docker_setup - script/fix_permissions.sh ${location}""", returnStatus: true) + return mapParallelStages } -def makeTarget(Map args = [:]) { - def context = args.context - def target = args.target - def directory = args.get('directory', '') - def clean = args.get('clean', true) - def withModule = args.get('withModule', false) - def directoryFlag = directory.trim() ? "-C ${directory}" : '' - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}") - whenTrue(clean) { - fixPermissions("${HOME}") - } +def cloud(Map args = [:]) { + node(args.label) { + startCloudTestEnv(name: args.directory, dirs: args.dirs) + } + withCloudTestEnv() { + try { + target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) + } finally { + terraformCleanup(name: args.directory, dir: args.directory) } } } -def mageTarget(Map args = [:]) { - def context = args.context - def directory = args.directory - def target = args.target - def withModule = args.get('withModule', false) - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") +def k8sTest(Map args = [:]) { + def versions = args.versions + node(args.label) { + versions.each{ v -> + stage("${args.context} ${v}"){ + withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ + withGithubNotify(context: "${args.context} ${v}") { + withBeatsEnv(archive: false, withModule: false) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") } + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") } + try { + sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") + sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") + sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") + } finally { + sh(label: 'Delete cluster', script: 'kind delete cluster') + } + } + } + } } } } } -def mageTargetWin(Map args = [:]) { +/** +* This method runs the given command supporting two kind of scenarios: +* - make -C then the dir(location) is not required, aka by disaling isMage: false +* - mage then the dir(location) is required, aka by enabling isMage: true. +*/ +def target(Map args = [:]) { def context = args.context - def directory = args.directory - def target = args.target + def command = args.command + def directory = args.get('directory', '') def withModule = args.get('withModule', false) - withGithubNotify(context: "${context}") { - withBeatsEnvWin(withModule: withModule, directory: directory) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMageWin() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") + def isMage = args.get('isMage', false) + node(args.label) { + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { + dumpVariables() + // make commands use -C while mage commands require the dir(folder) + // let's support this scenario with the location variable. + dir(isMage ? directory : '') { + cmd(label: "${command}", script: "${command}") + } } } } } -def getModulePattern(String toCompare) { - // Use contains to support the makeTarget(target: '-C ') while mageTarget(directory: '') - return (toCompare.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) -} - +/** +* This method wraps all the environment setup and pre-requirements to run any commands. +*/ def withBeatsEnv(Map args = [:], Closure body) { def archive = args.get('archive', true) def withModule = args.get('withModule', false) def directory = args.get('directory', '') - def modulePattern - if (withModule) { - modulePattern = getModulePattern(directory) + + def goRoot, path, magefile, pythonEnv, testResults, artifacts + + if(isUnix()) { + goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64" + path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}" + magefile = "${WORKSPACE}/.magefile" + pythonEnv = "${WORKSPACE}/python-env" + testResults = '**/build/TEST*.xml' + artifacts = '**/build/TEST*.out' + } else { + def chocoPath = 'C:\\ProgramData\\chocolatey\\bin' + def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' + goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" + path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}" + magefile = "${env.WORKSPACE}\\.magefile" + testResults = "**\\build\\TEST*.xml" + artifacts = "**\\build\\TEST*.out" } - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" deleteDir() unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' - + def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' withEnv([ - "HOME=${env.WORKSPACE}", + "DOCKER_PULL=0", "GOPATH=${env.WORKSPACE}", "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - "MAGEFILE_CACHE=${WORKSPACE}/.magefile", - "TEST_COVERAGE=true", + "HOME=${env.WORKSPACE}", + "MAGEFILE_CACHE=${magefile}", + "MODULE=${module}", + "PATH=${path}", + "PYTHON_ENV=${pythonEnv}", "RACE_DETECTOR=true", - "PYTHON_ENV=${WORKSPACE}/python-env", - "TEST_TAGS=${env.TEST_TAGS},oracle", - "DOCKER_PULL=0", - "MODULE=${module}" + "TEST_COVERAGE=true", + "TEST_TAGS=${env.TEST_TAGS},oracle" ]) { - if(isDockerInstalled()){ + if(isDockerInstalled()) { dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") } dir("${env.BASE_DIR}") { installTools() - // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. - // See https://github.com/elastic/beats/issues/17787. - setGitConfig() + if(isUnix()) { + // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. + // See https://github.com/elastic/beats/issues/17787. + sh(label: 'check git config', script: ''' + if [ -z "$(git config --get user.email)" ]; then + git config user.email "beatsmachine@users.noreply.github.com" + git config user.name "beatsmachine" + fi''') + } try { - if(!params.dry_run){ - body() - } + body() } finally { if (archive) { - archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out') + archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id) + } + // Tear down the setup for the permamnent workers. + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + fixPermissions("${WORKSPACE}") + deleteDir() } } } @@ -913,8 +288,57 @@ def withBeatsEnv(Map args = [:], Closure body) { } /** - This method archives and report the tests output, for such, it searches in certain folders - to bypass some issues when working with big repositories. +* This method fixes the filesystem permissions after the build has happenend. The reason is to +* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental +* issues. +*/ +def fixPermissions(location) { + if(isUnix()) { + sh(label: 'Fix permissions', script: """#!/usr/bin/env bash + set +x + source ./dev-tools/common.bash + docker_setup + script/fix_permissions.sh ${location}""", returnStatus: true) + } +} + +/** +* This method installs the required dependencies that are for some reason not available in the +* CI Workers. +*/ +def installTools() { + if(isUnix()) { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go/Mage/Python/Docker/Terraform ${GO_VERSION}", script: '.ci/scripts/install-tools.sh') } + } else { + retryWithSleep(retries: 2, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } + } +} + +/** +* This method gathers the module name, if required, in order to run the ITs only if +* the changeset affects a specific module. +* +* For such, it's required to look for changes under the module folder and exclude anything else +* such as asciidoc and png files. +*/ +def getCommonModuleInTheChangeSet(String directory) { + // Use contains to support the target(target: 'make -C ') while target(directory: '', target: '...') + def pattern = (directory.contains('x-pack') ? env.XPACK_MODULE_PATTERN : env.OSS_MODULE_PATTERN) + def module = '' + + // Transform folder structure in regex format since path separator is required to be escaped + def transformedDirectory = directory.replaceAll('/', '\\/') + def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" + def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" + dir("${env.BASE_DIR}") { + module = getGitMatchingGroup(pattern: pattern, exclude: exclude) + } + return module +} + +/** +* This method archives and report the tests output, for such, it searches in certain folders +* to bypass some issues when working with big repositories. */ def archiveTestOutput(Map args = [:]) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { @@ -923,219 +347,68 @@ def archiveTestOutput(Map args = [:]) { } cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults) + junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, id: args.id) archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") if (folder.trim()) { - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + goos() + def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS() tar(file: "${name}.tgz", archive: true, dir: folder) } } } } -def withBeatsEnvWin(Map args = [:], Closure body) { - def withModule = args.get('withModule', false) - def directory = args.get('directory', '') - def modulePattern - if (withModule) { - modulePattern = getModulePattern(directory) - } - final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' - final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' - def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64" - - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - - // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' - - withEnv([ - "HOME=${env.WORKSPACE}", - "GOPATH=${env.WORKSPACE}", - "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}", - "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", - "TEST_COVERAGE=true", - "RACE_DETECTOR=true", - "MODULE=${module}" - ]){ - dir("${env.BASE_DIR}"){ - installTools() - try { - if(!params.dry_run){ - body() - } - } finally { - archiveTestOutput(testResults: "**\\build\\TEST*.xml", artifacts: "**\\build\\TEST*.out") - } - } - } -} - -def installTools() { - def i = 2 // Number of retries - if(isUnix()) { - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") } - retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Mage", script: "make mage") } - } else { - retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } - } -} - -def goos(){ - def labels = env.NODE_LABELS - - if (labels.contains('linux')) { - return 'linux' - } else if (labels.contains('windows')) { - return 'windows' - } else if (labels.contains('darwin')) { - return 'darwin' - } - - throw new IllegalArgumentException("Unhandled OS name in NODE_LABELS: " + labels) -} - -def dumpMage(){ - echo "### MAGE DUMP ###" - sh(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpMageWin(){ - echo "### MAGE DUMP ###" - bat(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpFilteredEnvironment(){ - echo "### ENV DUMP ###" - echo "PATH: ${env.PATH}" - echo "HOME: ${env.HOME}" - echo "USERPROFILE: ${env.USERPROFILE}" - echo "BUILD_DIR: ${env.BUILD_DIR}" - echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" - echo "BEATS: ${env.BEATS}" - echo "PROJECTS: ${env.PROJECTS}" - echo "PROJECTS_ENV: ${env.PROJECTS_ENV}" - echo "PYTHON_ENV: ${env.PYTHON_ENV}" - echo "PYTHON_EXE: ${env.PYTHON_EXE}" - echo "PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE}" - echo "VENV_PARAMS: ${env.VENV_PARAMS}" - echo "FIND: ${env.FIND}" - echo "GOLINT: ${env.GOLINT}" - echo "GOLINT_REPO: ${env.GOLINT_REPO}" - echo "REVIEWDOG: ${env.REVIEWDOG}" - echo "REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS}" - echo "REVIEWDOG_REPO: ${env.REVIEWDOG_REPO}" - echo "XPACK_SUFFIX: ${env.XPACK_SUFFIX}" - echo "PKG_BUILD_DIR: ${env.PKG_BUILD_DIR}" - echo "PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR}" - echo "COVERAGE_TOOL: ${env.COVERAGE_TOOL}" - echo "COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO}" - echo "TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO}" - echo "NOW: ${env.NOW}" - echo "GOBUILD_FLAGS: ${env.GOBUILD_FLAGS}" - echo "GOIMPORTS: ${env.GOIMPORTS}" - echo "GOIMPORTS_REPO: ${env.GOIMPORTS_REPO}" - echo "GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX}" - echo "PROCESSES: ${env.PROCESSES}" - echo "TIMEOUT: ${env.TIMEOUT}" - echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" - echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" - echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}" - echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" - echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" - echo "STRESS_TESTS: ${env.STRESS_TESTS}" - echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" - echo "TEST_TAGS: ${env.TEST_TAGS}" - echo "GOX_OS: ${env.GOX_OS}" - echo "GOX_OSARCH: ${env.GOX_OSARCH}" - echo "GOX_FLAGS: ${env.GOX_FLAGS}" - echo "TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT}" - echo "BEAT_VERSION: ${env.BEAT_VERSION}" - echo "COMMIT_ID: ${env.COMMIT_ID}" - echo "DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME}" - echo "DOCKER_COMPOSE: ${env.DOCKER_COMPOSE}" - echo "DOCKER_CACHE: ${env.DOCKER_CACHE}" - echo "GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP}" - echo "PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS}" - echo "### END ENV DUMP ###" +/** +* This method wraps the junit built-in step to archive the test reports that gonna be populated later on +* with the runbld post build step. +*/ +def junitAndStore(Map args = [:]) { + junit(args) + // args.id could be null in some cases, so let's use the currentmilliseconds + def stageName = args.id ? args.id?.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" + stash(includes: args.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) + stashedTestReports[stageName] = stageName } -def k8sTest(versions){ - versions.each{ v -> - stage("k8s ${v}"){ - withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ - withGithubNotify(context: "K8s ${v}") { - withBeatsEnv(archive: false, withModule: false) { - sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") - sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") - sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") - sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") - sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") - sh(label: 'Delete cluster', script: 'kind delete cluster') +/** +* This method populates the test output using the runbld approach. For such it requires the +* global variable stashedTestReports. +* TODO: should be moved to the shared library +*/ +def runbld() { + catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { + if (stashedTestReports) { + def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats' + deleteDir() + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") + dir("${env.BASE_DIR}") { + // Unstash the test reports + stashedTestReports.each { k, v -> + dir(k) { + unstash(v) } } } + sh(label: 'Process JUnit reports with runbld', + script: """\ + ## for debugging purposes + find . -name "TEST-*.xml" + cat >./runbld-script </.*`. -* -* In addition, there are another two alternatives to report that there are -* changes, when `runAllStages` parameter is set to true or when running on a -* branch/tag basis. +* This method executes a closure with credentials for cloud test +* environments. */ -def isChanged(patterns){ - return ( - params.runAllStages // when runAllStages UI parameter is set to true - || !isPR() // when running on a branch/tag - || isGitRegionMatch(patterns: patterns, comparator: 'regexp') - ) -} - -def isChangedOSSCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^pytest.ini", - "^libbeat/.*", - "^testing/.*", - "^dev-tools/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -def isChangedXPackCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^pytest.ini", - "^libbeat/.*", - "^dev-tools/.*", - "^testing/.*", - "^x-pack/libbeat/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -// withCloudTestEnv executes a closure with credentials for cloud test -// environments. def withCloudTestEnv(Closure body) { def maskedVars = [] def testTags = "${env.TEST_TAGS}" @@ -1166,58 +439,60 @@ def withCloudTestEnv(Closure body) { } } -def terraformInit(String directory) { - dir(directory) { - sh(label: "Terraform Init on ${directory}", script: "terraform init") - } -} - -def terraformApply(String directory) { - terraformInit(directory) - dir(directory) { - sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") - } -} - -// Start testing environment on cloud using terraform. Terraform files are -// stashed so they can be used by other stages. They are also archived in -// case manual cleanup is needed. -// -// Example: -// startCloudTestEnv('x-pack-metricbeat', [ -// [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], -// ]) -// ... -// terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') -def startCloudTestEnv(String name, environments = []) { - withCloudTestEnv() { - withBeatsEnv(archive: false, withModule: false) { - def runAll = params.runAllCloudTests - try { - for (environment in environments) { - if (environment.cond || runAll) { +/** +* Start testing environment on cloud using terraform. Terraform files are +* stashed so they can be used by other stages. They are also archived in +* case manual cleanup is needed. +* +* Example: +* startCloudTestEnv(name: 'x-pack-metricbeat', dirs: ['x-pack/metricbeat/module/aws']) +* ... +* terraformCleanup(name: 'x-pack-metricbeat', dir: 'x-pack/metricbeat') +*/ +def startCloudTestEnv(Map args = [:]) { + String name = normalise(args.name) + def dirs = args.get('dirs',[]) + stage("${name}-prepare-cloud-env"){ + withCloudTestEnv() { + withBeatsEnv(archive: false, withModule: false) { + try { + for (folder in dirs) { retryWithSleep(retries: 2, seconds: 5, backoff: true){ - terraformApply(environment.dir) + terraformApply(folder) } } + } finally { + // Archive terraform states in case manual cleanup is needed. + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') } - } finally { - // Archive terraform states in case manual cleanup is needed. - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') + stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') } - stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') } } } +/** +* Run terraform in the given directory +*/ +def terraformApply(String directory) { + terraformInit(directory) + dir(directory) { + sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + } +} -// Looks for all terraform states in directory and runs terraform destroy for them, -// it uses terraform states previously stashed by startCloudTestEnv. -def terraformCleanup(String stashName, String directory) { - stage("Remove cloud scenarios in ${directory}"){ +/** +* Tear down the terraform environments, by looking for all terraform states in directory +* then it runs terraform destroy for each one. +* It uses terraform states previously stashed by startCloudTestEnv. +*/ +def terraformCleanup(Map args = [:]) { + String name = normalise(args.name) + String directory = args.dir + stage("${name}-tear-down-cloud-env"){ withCloudTestEnv() { withBeatsEnv(archive: false, withModule: false) { - unstash("terraform-${stashName}") + unstash("terraform-${name}") retryWithSleep(retries: 2, seconds: 5, backoff: true) { sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") } @@ -1226,184 +501,138 @@ def terraformCleanup(String stashName, String directory) { } } -def loadConfigEnvVars(){ - def empty = [] - env.GO_VERSION = readFile(".go-version").trim() - - withEnv(["HOME=${env.WORKSPACE}"]) { - retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") } - } - - // Libbeat is the core framework of Beats. It has no additional dependencies - // on other projects in the Beats repository. - env.BUILD_LIBBEAT = isChangedOSSCode(empty) - env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty) - - // Auditbeat depends on metricbeat as framework, but does not include any of - // the modules from Metricbeat. - // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. - env.BUILD_AUDITBEAT = isChangedOSSCode(getProjectDependencies('auditbeat')) - env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/auditbeat')) - - // Dockerlogbeat is a standalone Beat that only relies on libbeat. - env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/dockerlogbeat')) - - // Filebeat depends on libbeat only. - // The Filebeat x-pack build contains all functionality from OSS Filebeat. - env.BUILD_FILEBEAT = isChangedOSSCode(getProjectDependencies('filebeat')) - env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/filebeat')) - - // Metricbeat depends on libbeat only. - // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. - env.BUILD_METRICBEAT = isChangedOSSCode(getProjectDependencies('metricbeat')) - env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/metricbeat')) - - // Functionbeat is a standalone beat that depends on libbeat only. - // Functionbeat is available as x-pack build only. - env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/functionbeat')) - - // Heartbeat depends on libbeat only. - // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. - env.BUILD_HEARTBEAT = isChangedOSSCode(getProjectDependencies('heartbeat')) - env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/heartbeat')) - - // Journalbeat depends on libbeat only. - // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. - env.BUILD_JOURNALBEAT = isChangedOSSCode(getProjectDependencies('journalbeat')) - env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/journalbeat')) - - // Packetbeat depends on libbeat only. - // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. - env.BUILD_PACKETBEAT = isChangedOSSCode(getProjectDependencies('packetbeat')) - env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/packetbeat')) - - // Winlogbeat depends on libbeat only. - // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. - env.BUILD_WINLOGBEAT = isChangedOSSCode(getProjectDependencies('winlogbeat')) - env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/winlogbeat')) - - // Elastic-agent is a self-contained product, that depends on libbeat only. - // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. - // The agent is available as x-pack build only. - env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getProjectDependencies('x-pack/elastic-agent')) - - // The Kubernetes test use Filebeat and Metricbeat, but only need to be run - // if the deployment scripts have been updated. No Beats specific testing is - // involved. - env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) - - def generatorPatterns = ['^generator/.*'] - generatorPatterns.addAll(getProjectDependencies('generator/common/beatgen')) - generatorPatterns.addAll(getProjectDependencies('metricbeat/beater')) - env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) - - // Skip all the stages for changes only related to the documentation - env.ONLY_DOCS = isDocChangedOnly() - - // Enable macOS builds when required - env.BUILD_ON_MACOS = (params.macosTest // UI Input parameter is set to true - || !isPR() // For branches and tags - || matchesPrLabel(label: 'macOS') // If `macOS` GH label (Case-Sensitive) - || (env.GITHUB_COMMENT?.toLowerCase()?.contains('/test macos'))) // If `/test macos` in the GH comment (Case-Insensitive) -} - /** - This method gathers the module name, if required, in order to run the ITs only if - the changeset affects a specific module. - - For such, it's required to look for changes under the module folder and exclude anything else - such as ascidoc and png files. +* Prepare the terraform context in the given directory */ -def getCommonModuleInTheChangeSet(String pattern, String directory) { - def module = '' - // Transform folder structure in regex format since path separator is required to be escaped - def transformedDirectory = directory.replaceAll('/', '\\/') - def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" - def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" - dir("${env.BASE_DIR}") { - module = getGitMatchingGroup(pattern: pattern, exclude: exclude) +def terraformInit(String directory) { + dir(directory) { + sh(label: "Terraform Init on ${directory}", script: "terraform init") } - return module } /** - This method verifies if the changeset for the current pull request affect only changes related - to documentation, such as asciidoc and png files. +* Replace the slashes in the directory in case there are nested folders. */ -def isDocChangedOnly(){ - if (params.runAllStages || !env.CHANGE_ID?.trim()) { - log(level: 'INFO', text: 'Speed build for docs only is disabled for branches/tags or when forcing with the runAllStages parameter.') - return 'false' - } else { - log(level: "INFO", text: 'Check if the speed build for docs is enabled.') - return isGitRegionMatch(patterns: ['.*\\.(asciidoc|png)'], shouldMatchAll: true) - } +def normalise(String directory) { + return directory.replaceAll("[\\W]|_",'-') } /** - This method grab the dependencies of a Go module and transform them on regexp +* For debugging purposes. */ -def getProjectDependencies(beatName){ - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" - def output = "" - - withEnv([ - "HOME=${env.WORKSPACE}/${env.BASE_DIR}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - ]) { - output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - go list -deps ./${beatName} \ - | grep 'elastic/beats' \ - | sed -e "s#github.com/elastic/beats/v7/##g" \ - | awk '{print "^" \$1 "/.*"}' - """) - } - return output?.split('\n').collect{ item -> item as String } -} - -def setGitConfig(){ - sh(label: 'check git config', script: ''' - if [ -z "$(git config --get user.email)" ]; then - git config user.email "beatsmachine@users.noreply.github.com" - git config user.name "beatsmachine" - fi - ''') +def dumpVariables(){ + echo "### MAGE DUMP ###" + cmd(label: 'Dump mage variables', script: 'mage dumpVariables') + echo "### END MAGE DUMP ###" + echo """ + ### ENV DUMP ### + BEAT_VERSION: ${env.BEAT_VERSION} + BEATS: ${env.BEATS} + BUILD_DIR: ${env.BUILD_DIR} + COMMIT_ID: ${env.COMMIT_ID} + COVERAGE_DIR: ${env.COVERAGE_DIR} + COVERAGE_TOOL: ${env.COVERAGE_TOOL} + COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO} + DOCKER_CACHE: ${env.DOCKER_CACHE} + DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME} + DOCKER_COMPOSE: ${env.DOCKER_COMPOSE} + FIND: ${env.FIND} + GOBUILD_FLAGS: ${env.GOBUILD_FLAGS} + GOIMPORTS: ${env.GOIMPORTS} + GOIMPORTS_REPO: ${env.GOIMPORTS_REPO} + GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX} + GOLINT: ${env.GOLINT} + GOLINT_REPO: ${env.GOLINT_REPO} + GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP} + GOX_FLAGS: ${env.GOX_FLAGS} + GOX_OS: ${env.GOX_OS} + GOX_OSARCH: ${env.GOX_OSARCH} + HOME: ${env.HOME} + NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS} + NOW: ${env.NOW} + PATH: ${env.PATH} + PKG_BUILD_DIR: ${env.PKG_BUILD_DIR} + PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR} + PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS} + PROJECTS: ${env.PROJECTS} + PROJECTS_ENV: ${env.PROJECTS_ENV} + PYTHON_ENV: ${env.PYTHON_ENV} + PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE} + PYTHON_EXE: ${env.PYTHON_EXE} + PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES} + PROCESSES: ${env.PROCESSES} + REVIEWDOG: ${env.REVIEWDOG} + REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS} + REVIEWDOG_REPO: ${env.REVIEWDOG_REPO} + STRESS_TESTS: ${env.STRESS_TESTS} + STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS} + SYSTEM_TESTS: ${env.SYSTEM_TESTS} + TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO} + TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT} + TEST_TAGS: ${env.TEST_TAGS} + TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT} + TIMEOUT: ${env.TIMEOUT} + USERPROFILE: ${env.USERPROFILE} + VENV_PARAMS: ${env.VENV_PARAMS} + XPACK_SUFFIX: ${env.XPACK_SUFFIX} + ### END ENV DUMP ### + """ } def isDockerInstalled(){ - return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) + if (isUnix()) { + // TODO: some issues with macosx if(isInstalled(tool: 'docker', flag: '--version')) { + return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) + } else { + return false + } } -def junitAndStore(Map params = [:]){ - junit(params) - // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds - def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" - stash(includes: params.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) - stashedTestReports[stageName] = stageName +/** +* This class is the one used for running the parallel stages, therefore +* its arguments are passed by the beatsStages step. +* +* What parameters/arguments are supported: +* - label -> the worker labels +* - project -> the name of the project that should match with the folder name. +* - content -> the specific stage data in the /Jenkinsfile.yml +* - context -> the name of the stage, normally -(-)? +*/ +class RunCommand extends co.elastic.beats.BeatsFunction { + public RunCommand(Map args = [:]){ + super(args) + } + public run(Map args = [:]){ + def withModule = args.content.get('withModule', false) + if(args?.content?.containsKey('make')) { + steps.target(context: args.context, command: args.content.make, directory: args.project, label: args.label, withModule: withModule, isMage: false, id: args.id) + } + if(args?.content?.containsKey('mage')) { + steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) + } + if(args?.content?.containsKey('k8sTest')) { + steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id) + } + if(args?.content?.containsKey('cloud')) { + steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id) + } + } } -def runbld() { - catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { - if (stashedTestReports) { - def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats' - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${env.BASE_DIR}") { - // Unstash the test reports - stashedTestReports.each { k, v -> - dir(k) { - unstash(v) - } - } - } - sh(label: 'Process JUnit reports with runbld', - script: """\ - cat >./runbld-script < item as String } } } diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml new file mode 100644 index 00000000000..2f720bf055b --- /dev/null +++ b/Jenkinsfile.yml @@ -0,0 +1,47 @@ +projects: + - "auditbeat" + - "deploy/kubernetes" + - "filebeat" + - "generator" + - "heartbeat" + - "journalbeat" + - "libbeat" + - "metricbeat" + - "packetbeat" + - "winlogbeat" + - "x-pack/auditbeat" + - "x-pack/dockerlogbeat" + - "x-pack/elastic-agent" + - "x-pack/filebeat" + - "x-pack/functionbeat" + - "x-pack/libbeat" + - "x-pack/metricbeat" + - "x-pack/winlogbeat" + ##- "x-pack/heartbeat" It's not yet in the 1.0 pipeline. + ##- "x-pack/journalbeat" It's not yet in the 1.0 pipeline. + ##- "x-pack/packetbeat" It's not yet in the 1.0 pipeline. + +## Changeset macros that are defined here and used in each specific 2.0 pipeline. +changeset: + ci: + - "^Jenkinsfile" + - "^\\.ci/scripts/.*" + oss: + - "^go.mod" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + xpack: + - "^go.mod" + - "^dev-tools/.*" + - "^libbeat/.*" + - "^testing/.*" + - "^x-pack/libbeat/.*" + +## Proposal +## TBC: This will allow to configure what to do based on the PR configuration +disabled: + when: + labels: ## Skip the GitHub Pull Request builds if there is a GitHub label match + - "skip-ci" + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. diff --git a/README.md b/README.md index 00a0b8c9d7c..29272947f15 100644 --- a/README.md +++ b/README.md @@ -88,12 +88,16 @@ For testing purposes, we generate snapshot builds that you can find [here](https ## CI +### PR Comments + It is possible to trigger some jobs by putting a comment on a GitHub PR. (This service is only available for users affiliated with Elastic and not for open-source contributors.) * [beats][] * `jenkins run the tests please` or `jenkins run tests` or `/test` will kick off a default build. * `/test macos` will kick off a default build with also the `macos` stages. + * `/test ` will kick off the default build for the given PR in addition to the `` build itself. + * `/test for macos` will kick off a default build with also the `macos` stage for the ``. * [apm-beats-update][] * `/run apm-beats-update` * [apm-beats-packaging][] @@ -101,8 +105,14 @@ It is possible to trigger some jobs by putting a comment on a GitHub PR. * [apm-beats-tester][] * `/beats-tester` will kick of a build to validate the generated packages. +### PR Labels + +It's possible to configure the build on a GitHub PR by labelling the PR with the below labels + +* `` to force the following builds to run the stages for the `` +* `macOS` to force the following builds to run the `macos` stages. -[beats]: https://beats-ci.elastic.co/job/Beats/job/beats-beats-mbp/ +[beats]: https://beats-ci.elastic.co/job/Beats/job/beats/ [apm-beats-update]: https://beats-ci.elastic.co/job/Beats/job/apm-beats-update/ [apm-beats-packaging]: https://beats-ci.elastic.co/job/Beats/job/packaging/ [apm-beats-tester]: https://beats-ci.elastic.co/job/Beats/job/beats-tester/ diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..873e2c319f3 --- /dev/null +++ b/auditbeat/Jenkinsfile.yml @@ -0,0 +1,34 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test auditbeat" + labels: ## when PR labels matches any of those entries + - "auditbeat" + parameters: ## when parameter was selected in the UI. + - "auditbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + crosscompile: + make: "make -C auditbeat crosscompile" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/deploy/kubernetes/Jenkinsfile.yml b/deploy/kubernetes/Jenkinsfile.yml new file mode 100644 index 00000000000..452771edfb5 --- /dev/null +++ b/deploy/kubernetes/Jenkinsfile.yml @@ -0,0 +1,15 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^deploy/kubernetes/.*" + comments: ## when PR comment contains any of those entries + - "/test deploy/kubernetes" + labels: ## when PR labels matches any of those entries + - "kubernetes" + parameters: ## when parameter was selected in the UI. + - "kubernetes" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + k8sTest: + k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml new file mode 100644 index 00000000000..45b032accfb --- /dev/null +++ b/filebeat/Jenkinsfile.yml @@ -0,0 +1,33 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test filebeat" + labels: ## when PR labels matches any of those entries + - "filebeat" + parameters: ## when parameter was selected in the UI. + - "filebeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test filebeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml new file mode 100644 index 00000000000..071d24858bb --- /dev/null +++ b/generator/Jenkinsfile.yml @@ -0,0 +1,43 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^generator/.*" + - "#generator/common/beatgen" ## special token regarding the project dependency + - "#metricbeat/beater" ## special token regarding the project dependency + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test generator" + labels: ## when PR labels matches any of those entries + - "generator" + parameters: ## when parameter was selected in the UI. + - "generator" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + metricbeat-test: + make: "make -C generator/_templates/metricbeat test test-package" + beat-test: + make: "make -C generator/_templates/beat test test-package" + macos-metricbeat: + make: "make -C generator/_templates/metricbeat test" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test generator for macos" + labels: + - "macOS" + parameters: + - "macosTest" + macos-beat: + make: "make -C generator/_templates/beat test" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test generator for macos" + labels: + - "macOS" + parameters: + - "macosTest" diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..b8668715c3c --- /dev/null +++ b/heartbeat/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^heartbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test heartbeat" + labels: ## when PR labels matches any of those entries + - "heartbeat" + parameters: ## when parameter was selected in the UI. + - "heartbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test heartbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..12bb63f4cc6 --- /dev/null +++ b/journalbeat/Jenkinsfile.yml @@ -0,0 +1,17 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^journalbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test journalbeat" + labels: ## when PR labels matches any of those entries + - "journalbeat" + parameters: ## when parameter was selected in the UI. + - "journalbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + unitTest: + mage: "mage build unitTest" diff --git a/libbeat/Jenkinsfile.yml b/libbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..64a43269b13 --- /dev/null +++ b/libbeat/Jenkinsfile.yml @@ -0,0 +1,20 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test libbeat" + labels: ## when PR labels matches any of those entries + - "libbeat" + parameters: ## when parameter was selected in the UI. + - "libbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + crosscompile: + make: "make -C libbeat crosscompile" + stress-tests: + make: "make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests" diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..1219a27af77 --- /dev/null +++ b/metricbeat/Jenkinsfile.yml @@ -0,0 +1,40 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^metricbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test metricbeat" + labels: ## when PR labels matches any of those entries + - "metricbeat" + parameters: ## when parameter was selected in the UI. + - "metricbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + unitTest: + mage: "mage build unitTest" + goIntegTest: + mage: "mage goIntegTest" + withModule: true + pythonIntegTest: + mage: "mage pythonIntegTest" + withModule: true + crosscompile: + make: "make -C metricbeat crosscompile" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test metricbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..416e69a203b --- /dev/null +++ b/packetbeat/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^packetbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test packetbeat" + labels: ## when PR labels matches any of those entries + - "packetbeat" + parameters: ## when parameter was selected in the UI. + - "packetbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test packetbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..74eb55586d0 --- /dev/null +++ b/winlogbeat/Jenkinsfile.yml @@ -0,0 +1,21 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@oss" ## special token regarding the changeset for the oss + comments: ## when PR comment contains any of those entries + - "/test winlogbeat" + labels: ## when PR labels matches any of those entries + - "winlogbeat" + parameters: ## when parameter was selected in the UI. + - "winlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + crosscompile: + make: "make -C winlogbeat crosscompile" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..86f0832d3f2 --- /dev/null +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -0,0 +1,33 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/auditbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test auditbeat" + labels: ## when PR labels matches any of those entries + - "auditbeat" + parameters: ## when parameter was selected in the UI. + - "auditbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage update build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test auditbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..703bb3d66a9 --- /dev/null +++ b/x-pack/dockerlogbeat/Jenkinsfile.yml @@ -0,0 +1,18 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/dockerlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/dockerlogbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-dockerlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-dockerlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml new file mode 100644 index 00000000000..8f99e11da3c --- /dev/null +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/elastic-agent/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/elastic-agent" + labels: ## when PR labels matches any of those entries + - "x-pack-elastic-agent" + parameters: ## when parameter was selected in the UI. + - "x-pack-elastic-agent" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/elastic-agent for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml new file mode 100644 index 00000000000..d3d5e6d862e --- /dev/null +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -0,0 +1,33 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/filebeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/filebeat" + labels: ## when PR labels matches any of those entries + - "x-pack-filebeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-filebeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/filebeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..f3428ae7cc8 --- /dev/null +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -0,0 +1,32 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/functionbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/functionbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-functionbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-functionbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test && GO_VERSION=1.13.1 mage testGCPFunctions" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/functionbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/libbeat/Jenkinsfile.yml b/x-pack/libbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..87019f071a0 --- /dev/null +++ b/x-pack/libbeat/Jenkinsfile.yml @@ -0,0 +1,17 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/libbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/libbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-libbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-libbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + mage: "mage build test" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..2448d43d85b --- /dev/null +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -0,0 +1,35 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/metricbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/metricbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-metricbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-metricbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + build: + cloud: "mage build test" + withModule: true ## run the ITs only if the changeset affects a specific module. + dirs: ## run the cloud tests for the given modules. + - "x-pack/metricbeat/module/aws" + macos: + mage: "mage build unitTest" + platforms: ## override default label in this specific stage. + - "macosx" + when: ## Aggregate when with the top-level one. + comments: + - "/test x-pack/metricbeat for macos" + labels: + - "macOS" + parameters: + - "macosTest" + windows: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..8496265e0ac --- /dev/null +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -0,0 +1,20 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/winlogbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-winlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-winlogbeat" + tags: true ## for all the tags +platform: "linux && ubuntu-18" ## default label for all the stages +stages: + windows: + mage: "mage build unitTest" + withModule: true + platforms: ## override default labels in this specific stage. + - "windows-2019" diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml new file mode 100644 index 00000000000..396d1f03a7c --- /dev/null +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -0,0 +1,20 @@ +when: + branches: true ## for all the branches + changeset: ## when PR contains any of those entries in the changeset + - "^x-pack/winlogbeat/.*" + - "@ci" ## special token regarding the changeset for the ci + - "@xpack" ## special token regarding the changeset for the xpack + comments: ## when PR comment contains any of those entries + - "/test x-pack/winlogbeat" + labels: ## when PR labels matches any of those entries + - "x-pack-winlogbeat" + parameters: ## when parameter was selected in the UI. + - "x-pack-winlogbeat" + tags: true ## for all the tags +platform: "windows-2019" ## default label for all the stages +stages: + build: + mage: "mage build unitTest" + withModule: true + platforms: ## override default labels in this specific stage. + - "windows-2019"