Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[7.x] feat: filter changes using go list output (#17397) #17747

Closed
wants to merge 2 commits into from
Closed

[7.x] feat: filter changes using go list output (#17397) #17747

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c5eca40
feat: filter changes using go list output (#17397)
kuisathaverat Apr 16, 2020
15d5361
Update Jenkinsfile
kuisathaverat Apr 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 57 additions & 53 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ pipeline {
BASE_DIR = 'src/github.com/elastic/beats'
GOX_FLAGS = "-arch amd64"
DOCKER_COMPOSE_VERSION = "1.21.0"
PIPELINE_LOG_LEVEL = "INFO"
}
options {
timeout(time: 2, unit: 'HOURS')
Expand All @@ -17,7 +18,6 @@ pipeline {
disableResume()
durabilityHint('PERFORMANCE_OPTIMIZED')
disableConcurrentBuilds()
// checkoutToSubdirectory "${env.BASE_DIR}"
}
triggers {
issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
Expand All @@ -27,6 +27,7 @@ pipeline {
booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.')
booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps')
booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow')
}
stages {
/**
Expand Down Expand Up @@ -662,7 +663,9 @@ def withBeatsEnv(boolean archive, Closure body) {
sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh")
sh(label: "Install Mage", script: "make mage")
try {
body()
if(!params.dry_run){
body()
}
} finally {
if (archive) {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
Expand Down Expand Up @@ -695,7 +698,9 @@ def withBeatsEnvWin(Closure body) {
dir("${env.BASE_DIR}"){
bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat")
try {
body()
if(!params.dry_run){
body()
}
} finally {
catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
junit(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml")
Expand Down Expand Up @@ -831,19 +836,21 @@ def isChanged(patterns){
}

def isChangedOSSCode(patterns) {
def always = [
def allPatterns = [
"^Jenkinsfile",
"^vendor/.*",
"^libbeat/.*",
"^testing/.*",
"^dev-tools/.*",
"^\\.ci/.*",
]
return isChanged(always + patterns)
allPatterns.addAll(patterns)
allPatterns.addAll(getVendorPatterns('libbeat'))
return isChanged(allPatterns)
}

def isChangedXPackCode(patterns) {
def always = [
def allPatterns = [
"^Jenkinsfile",
"^vendor/.*",
"^libbeat/.*",
Expand All @@ -852,91 +859,88 @@ def isChangedXPackCode(patterns) {
"^x-pack/libbeat/.*",
"^\\.ci/.*",
]
return isChanged(always + patterns)
allPatterns.addAll(patterns)
allPatterns.addAll(getVendorPatterns('x-pack/libbeat'))
return isChanged(allPatterns)
}

def loadConfigEnvVars(){
def empty = []
env.GO_VERSION = readFile(".go-version").trim()

// Libbeat is the core framework of Beats. It has no additional dependencies
// on other projects in the Beats repository.
env.BUILD_LIBBEAT = isChangedOSSCode([])
env.BUILD_LIBBEAT_XPACK = env.BUILD_LIBBEAT || isChangedXPackCode([])
env.BUILD_LIBBEAT = isChangedOSSCode(empty)
env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty)

// Auditbeat depends on metricbeat as framework, but does not include any of
// the modules from Metricbeat.
// The Auditbeat x-pack build contains all functionality from OSS Auditbeat.
env.BUILD_AUDITBEAT = isChangedOSSCode([
"^metricbeat/.*",
"^auditbeat/.*",
])
env.BUILD_AUDITBEAT_XPACK = env.BUILD_AUDITBEAT || isChangedXPackCode([
"^x-pack/auditbeat/.*",
])
env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat'))
env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat'))

// Dockerlogbeat is a standalone Beat that only relies on libbeat.
env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode([
"^x-pack/dockerlogbeat/.*",
])
env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat'))

// Filebeat depends on libbeat only.
// The Filebeat x-pack build contains all functionality from OSS Filebeat.
env.BUILD_FILEBEAT = isChangedOSSCode(["^filebeat/.*"])
env.BUILD_FILEBEAT_XPACK = env.BUILD_FILEBEAT || isChangedXPackCode([
"^x-pack/filebeat/.*",
])
env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat'))
env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat'))

// Metricbeat depends on libbeat only.
// The Metricbeat x-pack build contains all functionality from OSS Metricbeat.
env.BUILD_METRICBEAT = isChangedOSSCode(["^metricbeat/.*"])
env.BUILD_METRICBEAT_XPACK = env.BUILD_METRICBEAT || isChangedXPackCode([
"^x-pack/metricbeat/.*",
])
env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat'))
env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat'))

// Functionbeat is a standalone beat that depends on libbeat only.
// Functionbeat is available as x-pack build only.
env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode([
"^x-pack/functionbeat/.*",
])
env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat'))

// Heartbeat depends on libbeat only.
// The Heartbeat x-pack build contains all functionality from OSS Heartbeat.
env.BUILD_HEARTBEAT = isChangedOSSCode(["^heartbeat/.*"])
env.BUILD_HEARTBEAT_XPACK = env.BUILD_HEARTBEAT || isChangedXPackCode([
"^x-pack/heartbeat/.*",
])

env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat'))
env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat'))

// Journalbeat depends on libbeat only.
// The Journalbeat x-pack build contains all functionality from OSS Journalbeat.
env.BUILD_JOURNALBEAT = isChangedOSSCode(["^journalbeat/.*"])
env.BUILD_JOURNALBEAT_XPACK = env.BUILD_JOURNALBEAT || isChangedXPackCode([
"^x-pack/journalbeat/.*",
])
env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat'))
env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat'))

// Packetbeat depends on libbeat only.
// The Packetbeat x-pack build contains all functionality from OSS Packetbeat.
env.BUILD_PACKETBEAT = isChangedOSSCode(["^packetbeat/.*"])
env.BUILD_PACKETBEAT_XPACK = env.BUILD_PACKETBEAT || isChangedXPackCode([
"^x-pack/packetbeat/.*",
])
env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat'))
env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat'))

// Winlogbeat depends on libbeat only.
// The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat.
env.BUILD_WINLOGBEAT = isChangedOSSCode(["^winlogbeat/.*"])
env.BUILD_WINLOGBEAT_XPACK = env.BUILD_WINLOGBEAT || isChangedXPackCode([
"^x-pack/winlogbeat/.*",
])
env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat'))
env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat'))

// Elastic-agent is a self-contained product, that depends on libbeat only.
// The agent acts as a supervisor for other Beats like Filebeat or Metricbeat.
// The agent is available as x-pack build only.
env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/elastic-agent'))

// The Kubernetes test use Filebeat and Metricbeat, but only need to be run
// if the deployment scripts have been updated. No Beats specific testing is
// involved.
env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"])

env.BUILD_GENERATOR = isChangedOSSCode(["^generator/.*"])

env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode([
"^x-pack/elastic-agent/.*",
])
env.BUILD_GENERATOR = isChangedOSSCode(getVendorPatterns('generator'))
}

env.GO_VERSION = readFile(".go-version").trim()
/**
This method grab the dependencies of a Go module and transform them on regexp
*/
def getVendorPatterns(beatName){
def output = ""
docker.image("golang:${GO_VERSION}").inside{
output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """
export HOME=${WORKSPACE}/${BASE_DIR}
go list -mod=vendor -f '{{ .ImportPath }}{{ "\\n" }}{{ join .Deps "\\n" }}' ./${beatName}\
|awk '{print \$1"/.*"}'\
|sed -e "s#github.com/elastic/beats/v7/##g"
""")
}
return output?.split('\n').collect{ item -> item as String }
}