From 208c0697338d495922d6f09ce0c693cfc04a4998 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Tue, 14 Apr 2020 09:55:13 +0200 Subject: [PATCH 1/6] fix: login into the docker registry (#17620) * fix: login into the docker registry * test: make a pull after login to test * docs: add note to the workaround --- Jenkinsfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index d480d2ee396..d7adadeae06 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,8 @@ pipeline { BASE_DIR = 'src/github.com/elastic/beats' GOX_FLAGS = "-arch amd64" DOCKER_COMPOSE_VERSION = "1.21.0" + DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' + DOCKER_REGISTRY = 'docker.elastic.co' } options { timeout(time: 2, unit: 'HOURS') @@ -644,6 +646,9 @@ def withBeatsEnv(boolean archive, Closure body) { ]) { deleteDir() unstash 'source' + dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + // FIXME workaround untill we fix the packer cache + sh 'docker pull docker.elastic.co/observability-ci/database-enterprise:12.2.0.1' dir("${env.BASE_DIR}") { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") From fb32fbdd0edd1b856daad8a31b9e8db5aa479b80 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 16 Apr 2020 10:28:49 +0200 Subject: [PATCH 2/6] feat: filter changes using go list output (#17397) * feat: filter changes using go list output * fix: auditbeat trigger --- Jenkinsfile | 110 +++++++++++++++++++++++++++------------------------- 1 file changed, 57 insertions(+), 53 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index d7adadeae06..fe0a3443413 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,6 +8,7 @@ pipeline { BASE_DIR = 'src/github.com/elastic/beats' GOX_FLAGS = "-arch amd64" DOCKER_COMPOSE_VERSION = "1.21.0" + PIPELINE_LOG_LEVEL = "INFO" DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_REGISTRY = 'docker.elastic.co' } @@ -19,7 +20,6 @@ pipeline { disableResume() durabilityHint('PERFORMANCE_OPTIMIZED') disableConcurrentBuilds() -// checkoutToSubdirectory "${env.BASE_DIR}" } triggers { issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') @@ -29,6 +29,7 @@ pipeline { booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.') booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') + booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow') } stages { /** @@ -654,7 +655,9 @@ def withBeatsEnv(boolean archive, Closure body) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") sh(label: "Install Mage", script: "make mage") try { - body() + if(!params.dry_run){ + body() + } } finally { if (archive) { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { @@ -687,7 +690,9 @@ def withBeatsEnvWin(Closure body) { dir("${env.BASE_DIR}"){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") try { - body() + if(!params.dry_run){ + body() + } } finally { catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") @@ -823,7 +828,7 @@ def isChanged(patterns){ } def isChangedOSSCode(patterns) { - def always = [ + def allPatterns = [ "^Jenkinsfile", "^vendor/.*", "^libbeat/.*", @@ -831,11 +836,13 @@ def isChangedOSSCode(patterns) { "^dev-tools/.*", "^\\.ci/.*", ] - return isChanged(always + patterns) + allPatterns.addAll(patterns) + allPatterns.addAll(getVendorPatterns('libbeat')) + return isChanged(allPatterns) } def isChangedXPackCode(patterns) { - def always = [ + def allPatterns = [ "^Jenkinsfile", "^vendor/.*", "^libbeat/.*", @@ -844,91 +851,88 @@ def isChangedXPackCode(patterns) { "^x-pack/libbeat/.*", "^\\.ci/.*", ] - return isChanged(always + patterns) + allPatterns.addAll(patterns) + allPatterns.addAll(getVendorPatterns('x-pack/libbeat')) + return isChanged(allPatterns) } def loadConfigEnvVars(){ + def empty = [] + env.GO_VERSION = readFile(".go-version").trim() // Libbeat is the core framework of Beats. It has no additional dependencies // on other projects in the Beats repository. - env.BUILD_LIBBEAT = isChangedOSSCode([]) - env.BUILD_LIBBEAT_XPACK = env.BUILD_LIBBEAT || isChangedXPackCode([]) + env.BUILD_LIBBEAT = isChangedOSSCode(empty) + env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty) // Auditbeat depends on metricbeat as framework, but does not include any of // the modules from Metricbeat. // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. - env.BUILD_AUDITBEAT = isChangedOSSCode([ - "^metricbeat/.*", - "^auditbeat/.*", - ]) - env.BUILD_AUDITBEAT_XPACK = env.BUILD_AUDITBEAT || isChangedXPackCode([ - "^x-pack/auditbeat/.*", - ]) + env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat')) + env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat')) // Dockerlogbeat is a standalone Beat that only relies on libbeat. - env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode([ - "^x-pack/dockerlogbeat/.*", - ]) + env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat')) // Filebeat depends on libbeat only. // The Filebeat x-pack build contains all functionality from OSS Filebeat. - env.BUILD_FILEBEAT = isChangedOSSCode(["^filebeat/.*"]) - env.BUILD_FILEBEAT_XPACK = env.BUILD_FILEBEAT || isChangedXPackCode([ - "^x-pack/filebeat/.*", - ]) + env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat')) + env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat')) // Metricbeat depends on libbeat only. // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. - env.BUILD_METRICBEAT = isChangedOSSCode(["^metricbeat/.*"]) - env.BUILD_METRICBEAT_XPACK = env.BUILD_METRICBEAT || isChangedXPackCode([ - "^x-pack/metricbeat/.*", - ]) + env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat')) + env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat')) // Functionbeat is a standalone beat that depends on libbeat only. // Functionbeat is available as x-pack build only. - env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode([ - "^x-pack/functionbeat/.*", - ]) + env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat')) // Heartbeat depends on libbeat only. // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. - env.BUILD_HEARTBEAT = isChangedOSSCode(["^heartbeat/.*"]) - env.BUILD_HEARTBEAT_XPACK = env.BUILD_HEARTBEAT || isChangedXPackCode([ - "^x-pack/heartbeat/.*", - ]) - + env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat')) + env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat')) // Journalbeat depends on libbeat only. // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. - env.BUILD_JOURNALBEAT = isChangedOSSCode(["^journalbeat/.*"]) - env.BUILD_JOURNALBEAT_XPACK = env.BUILD_JOURNALBEAT || isChangedXPackCode([ - "^x-pack/journalbeat/.*", - ]) + env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat')) + env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat')) // Packetbeat depends on libbeat only. // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. - env.BUILD_PACKETBEAT = isChangedOSSCode(["^packetbeat/.*"]) - env.BUILD_PACKETBEAT_XPACK = env.BUILD_PACKETBEAT || isChangedXPackCode([ - "^x-pack/packetbeat/.*", - ]) + env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat')) + env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat')) // Winlogbeat depends on libbeat only. // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. - env.BUILD_WINLOGBEAT = isChangedOSSCode(["^winlogbeat/.*"]) - env.BUILD_WINLOGBEAT_XPACK = env.BUILD_WINLOGBEAT || isChangedXPackCode([ - "^x-pack/winlogbeat/.*", - ]) + env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat')) + env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat')) + + // Elastic-agent is a self-contained product, that depends on libbeat only. + // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. + // The agent is available as x-pack build only. + env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/elastic-agent')) // The Kubernetes test use Filebeat and Metricbeat, but only need to be run // if the deployment scripts have been updated. No Beats specific testing is // involved. env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) - env.BUILD_GENERATOR = isChangedOSSCode(["^generator/.*"]) - - env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode([ - "^x-pack/elastic-agent/.*", - ]) + env.BUILD_GENERATOR = isChangedOSSCode(getVendorPatterns('generator')) +} - env.GO_VERSION = readFile(".go-version").trim() +/** + This method grab the dependencies of a Go module and transform them on regexp +*/ +def getVendorPatterns(beatName){ + def output = "" + docker.image("golang:${GO_VERSION}").inside{ + output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ + export HOME=${WORKSPACE}/${BASE_DIR} + go list -mod=vendor -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\ + |awk '{print \$1"/.*"}'\ + |sed -e "s#github.com/elastic/beats/v7/##g" + """) + } + return output?.split('\n').collect{ item -> item as String } } From 42e356cf10ead9dc7ecd0954d520ed3af7cfe2da Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 17 Apr 2020 09:29:30 +0200 Subject: [PATCH 3/6] fix: disable workaround on macos (#17750) * fix: disable workaround on macos * Update Jenkinsfile Co-Authored-By: Victor Martinez * Update Jenkinsfile Co-Authored-By: Andrew Kroh Co-authored-by: Victor Martinez Co-authored-by: Andrew Kroh --- Jenkinsfile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fe0a3443413..973cefecd25 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -647,9 +647,15 @@ def withBeatsEnv(boolean archive, Closure body) { ]) { deleteDir() unstash 'source' - dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - // FIXME workaround untill we fix the packer cache - sh 'docker pull docker.elastic.co/observability-ci/database-enterprise:12.2.0.1' + if(os == 'linux'){ + dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") + // FIXME workaround until we fix the packer cache + // Retry to avoid DDoS detection from the server + retry(3) { + sleep randomNumber(min: 2, max: 5) + sh 'docker pull docker.elastic.co/observability-ci/database-enterprise:12.2.0.1' + } + } dir("${env.BASE_DIR}") { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") From ba63fb3c87b4376ceb8ae74bcd49e8587b5e3550 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Fri, 17 Apr 2020 14:16:12 +0200 Subject: [PATCH 4/6] ci: set git user configuration if it is not set (#17782) * ci: set git user configuration if it is not set * Apply suggestions from code review --- Jenkinsfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 973cefecd25..e7cd17ddf71 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -660,6 +660,9 @@ def withBeatsEnv(boolean archive, Closure body) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") sh(label: "Install Mage", script: "make mage") + // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. + // See https://github.com/elastic/beats/issues/17787. + setGitConfig() try { if(!params.dry_run){ body() @@ -942,3 +945,12 @@ def getVendorPatterns(beatName){ } return output?.split('\n').collect{ item -> item as String } } + +def setGitConfig(){ + sh(label: 'check git config', script: ''' + if [ -z "$(git config --get user.email)" ]; then + git config user.email "beatsmachine@users.noreply.github.com" + git config user.name "beatsmachine" + fi + ''') +} From 5edd8d73617588d39adaf12be7a04394bf9b86a7 Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 23 Apr 2020 11:11:35 +0200 Subject: [PATCH 5/6] fix: mount Docker credentials (#17798) * fix: mount Docker credentials * Apply suggestions from code review * Apply suggestions from code review --- Jenkinsfile | 12 +++++------- x-pack/metricbeat/docker-compose.yml | 1 + 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e7cd17ddf71..532e97ea0a2 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -647,14 +647,8 @@ def withBeatsEnv(boolean archive, Closure body) { ]) { deleteDir() unstash 'source' - if(os == 'linux'){ + if(isDockerInstalled()){ dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - // FIXME workaround until we fix the packer cache - // Retry to avoid DDoS detection from the server - retry(3) { - sleep randomNumber(min: 2, max: 5) - sh 'docker pull docker.elastic.co/observability-ci/database-enterprise:12.2.0.1' - } } dir("${env.BASE_DIR}") { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") @@ -954,3 +948,7 @@ def setGitConfig(){ fi ''') } + +def isDockerInstalled(){ + return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) +} diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml index c89b558605d..83ac016ffd7 100644 --- a/x-pack/metricbeat/docker-compose.yml +++ b/x-pack/metricbeat/docker-compose.yml @@ -10,6 +10,7 @@ services: volumes: - ${PWD}/../..:/go/src/github.com/elastic/beats/ - /var/run/docker.sock:/var/run/docker.sock + - ${HOME}/.docker:/root/.docker:ro network_mode: host command: make From 6391d8e34e1c0d7cb92ca8666a338ff8013787af Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Tue, 28 Apr 2020 10:45:43 +0200 Subject: [PATCH 6/6] Review dependency patterns collection in Jenkins (#18004) Fix dependency checking for generators, looking now for dependencies of the generators code itself (beatgen) and Metricbeat beater. Stop looking for dependencies of libbeat in all beats, this is redundant as they are already transitive dependencies. Run collection of dependecies in the host instead of using docker, what is slightly faster. --- Jenkinsfile | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 532e97ea0a2..1086f730ea7 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,13 +40,13 @@ pipeline { steps { deleteDir() gitCheckout(basedir: "${BASE_DIR}") + stash allowEmpty: true, name: 'source', useDefaultExcludes: false dir("${BASE_DIR}"){ loadConfigEnvVars() } whenTrue(params.debug){ dumpFilteredEnvironment() } - stash allowEmpty: true, name: 'source', useDefaultExcludes: false } } stage('Lint'){ @@ -840,7 +840,6 @@ def isChangedOSSCode(patterns) { "^\\.ci/.*", ] allPatterns.addAll(patterns) - allPatterns.addAll(getVendorPatterns('libbeat')) return isChanged(allPatterns) } @@ -855,7 +854,6 @@ def isChangedXPackCode(patterns) { "^\\.ci/.*", ] allPatterns.addAll(patterns) - allPatterns.addAll(getVendorPatterns('x-pack/libbeat')) return isChanged(allPatterns) } @@ -863,6 +861,10 @@ def loadConfigEnvVars(){ def empty = [] env.GO_VERSION = readFile(".go-version").trim() + withEnv(["HOME=${env.WORKSPACE}"]) { + sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") + } + // Libbeat is the core framework of Beats. It has no additional dependencies // on other projects in the Beats repository. env.BUILD_LIBBEAT = isChangedOSSCode(empty) @@ -921,17 +923,25 @@ def loadConfigEnvVars(){ // involved. env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) - env.BUILD_GENERATOR = isChangedOSSCode(getVendorPatterns('generator')) + def generatorPatterns = ['^generator/.*'] + generatorPatterns.addAll(getVendorPatterns('generator/common/beatgen')) + generatorPatterns.addAll(getVendorPatterns('metricbeat/beater')) + env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) } /** This method grab the dependencies of a Go module and transform them on regexp */ def getVendorPatterns(beatName){ + def os = goos() + def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" def output = "" - docker.image("golang:${GO_VERSION}").inside{ + + withEnv([ + "HOME=${env.WORKSPACE}/${env.BASE_DIR}", + "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", + ]) { output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - export HOME=${WORKSPACE}/${BASE_DIR} go list -mod=vendor -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\ |awk '{print \$1"/.*"}'\ |sed -e "s#github.com/elastic/beats/v7/##g"