diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml index c60242b944..d72667765d 100644 --- a/.ci/.e2e-tests.yaml +++ b/.ci/.e2e-tests.yaml @@ -1,24 +1,24 @@ --- SUITES: - suite: "helm" - feature: "apm-server" + tags: "apm-server" - suite: "helm" - feature: "filebeat" + tags: "filebeat" - suite: "helm" - feature: "metricbeat" + tags: "metricbeat" - suite: "ingest-manager" - feature: "agent_endpoint_integration" + tags: "agent_endpoint_integration" - suite: "ingest-manager" - feature: "stand_alone_mode" + tags: "stand_alone_mode" - suite: "ingest-manager" - feature: "fleet_mode" + tags: "fleet_mode" - suite: "metricbeat" - feature: "apache" + tags: "apache" - suite: "metricbeat" - feature: "metricbeat" + tags: "metricbeat" - suite: "metricbeat" - feature: "mysql" + tags: "mysql" - suite: "metricbeat" - feature: "redis" + tags: "redis" - suite: "metricbeat" - feature: "vsphere" + tags: "vsphere" diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index db39f601f7..c5a052777f 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -27,7 +27,7 @@ pipeline { issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') } parameters { - choice(name: 'runTestsSuite', choices: ['all', 'helm', 'ingest-manager', 'metricbeat'], description: 'Choose which test suite to run (default: all)') + string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)') booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources") booleanParam(name: "forceSkipPresubmit", defaultValue: false, description: "If it's needed to execute the pre-submit tests: unit and precommit.") booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify with green builds.") @@ -184,19 +184,20 @@ pipeline { unstash 'source' dir("${BASE_DIR}") { script { - def suiteParam = params.runTestsSuite - def suites = readYaml(file: '.ci/.e2e-tests.yaml') + def suitesParam = params.runTestsSuites + def existingSuites = readYaml(file: '.ci/.e2e-tests.yaml') def parallelTasks = [:] - suites['SUITES'].each { item -> - def suite = item.suite - def feature = item.feature - if (suiteParam == "all" || suiteParam == suite) { - def regexps = [ "^e2e/_suites/${suite}/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go" ] - if ("${FORCE_SKIP_GIT_CHECKS}" == "true" || isGitRegionMatch(patterns: regexps, shouldMatchAll: false)) { - log(level: 'INFO', text: "Adding ${suite}:${feature} test suite to the build execution") - parallelTasks["${suite}_${feature}"] = generateFunctionalTestStep(suite: "${suite}", feature: "${feature}") - } else { - log(level: 'WARN', text: "The ${suite}:${feature} test suite won't be executed because there are no modified files") + + if (suitesParam == "") { + log(level: 'DEBUG', text: "Iterate through existing test suites") + existingSuites['SUITES'].each { item -> + checkTestSuite(parallelTasks, item.suite, item.tags) + } + } else { + log(level: 'DEBUG', text: "Iterate through the comma-separated test suites (${suitesParam}), comparing with the existing test suites") + suitesParam.split(',').each { suiteParam -> + existingSuites['SUITES'].findAll { suiteParam.trim() == it.suite }.each { item -> + checkTestSuite(parallelTasks, item.suite, item.tags) } } } @@ -236,24 +237,47 @@ pipeline { } success { whenTrue(!isPR() && params.notifyOnGreenBuilds) { - slackSend(channel: "#${env.SLACK_CHANNEL}", color: 'good', - message: "[${params.runTestsSuite}] Build Passed: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.RUN_DISPLAY_URL}|Open>).", - tokenCredentialId: 'jenkins-slack-integration-token') + doSlackSend('good') } } unsuccessful { whenFalse(isPR()) { - slackSend(channel: "#${env.SLACK_CHANNEL}", color: 'danger', - message: "[${params.runTestsSuite}] Build Failed: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.RUN_DISPLAY_URL}|Open>).", - tokenCredentialId: 'jenkins-slack-integration-token') + doSlackSend('danger') } } } } -def generateStep(Map params = [:]){ - def oss = params.get('oss') - def platform = params.get('platform') +def checkTestSuite(Map parallelTasks = [:], String suite, String tags) { + def regexps = [ "^e2e/_suites/${suite}/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go" ] + if ("${FORCE_SKIP_GIT_CHECKS}" == "true" || isGitRegionMatch(patterns: regexps, shouldMatchAll: false)) { + log(level: 'INFO', text: "Adding ${suite}:${tags} test suite to the build execution") + parallelTasks["${suite}_${tags}"] = generateFunctionalTestStep(suite: "${suite}", tags: "${tags}") + } else { + log(level: 'WARN', text: "The ${suite}:${tags} test suite won't be executed because there are no modified files") + } +} + +def doSlackSend(String color) { + def buildStatus = "Failed" + if (color == "good") { + buildStatus = "Passed" + } + + def testSuites = "${params.runTestsSuites}" + if (testSuites == "") { + testsSuites = "All suites" + } + + def message = "*[${testsSuites}]* Build `${buildStatus}`: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.RUN_DISPLAY_URL}|Open>)." + + slackSend(channel: "#${env.SLACK_CHANNEL}", color: "${color}", message: "${message}",tokenCredentialId: 'jenkins-slack-integration-token') +} + + +def generateStep(Map args = [:]){ + def oss = args.get('oss') + def platform = args.get('platform') return { node('ubuntu-18.04 && immutable && docker') { try { @@ -273,11 +297,11 @@ def generateStep(Map params = [:]){ } } -def generateFunctionalTestStep(Map params = [:]){ - def suite = params.get('suite') +def generateFunctionalTestStep(Map args = [:]){ + def suite = args.get('suite') def sneakCaseSuite = suite.toUpperCase().replaceAll("-", "_") def stackVersion = env."${sneakCaseSuite}_STACK_VERSION" - def feature = params.get('feature') + def tags = args.get('tags') return { node('ubuntu-18.04 && immutable && docker') { try { @@ -289,11 +313,11 @@ def generateFunctionalTestStep(Map params = [:]){ } retry(3){ dir("${BASE_DIR}"){ - sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${feature}" + sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}" } } dir("${BASE_DIR}"){ - sh script: """.ci/scripts/functional-test.sh "${suite}" "${feature}" "${stackVersion}" "${METRICBEAT_VERSION}" """, label: "Run functional tests for ${suite}:${feature}" + sh script: """.ci/scripts/functional-test.sh "${suite}" "${tags}" "${stackVersion}" "${METRICBEAT_VERSION}" """, label: "Run functional tests for ${suite}:${tags}" } } } catch(e) { diff --git a/.ci/e2eTestingHelmDaily.groovy b/.ci/e2eTestingHelmDaily.groovy index 99cd00874f..206713202b 100644 --- a/.ci/e2eTestingHelmDaily.groovy +++ b/.ci/e2eTestingHelmDaily.groovy @@ -44,7 +44,7 @@ pipeline { parameters: [ booleanParam(name: 'forceSkipGitChecks', value: true), booleanParam(name: 'forceSkipPresubmit', value: true), - string(name: 'runTestsSuite', value: 'helm'), + string(name: 'runTestsSuites', value: 'helm'), string(name: 'SLACK_CHANNEL', value: "integrations"), ], propagate: false, diff --git a/.ci/e2eTestingIngestManagerDaily.groovy b/.ci/e2eTestingIngestManagerDaily.groovy index 7d7cb5c4ae..d5ba4ece95 100644 --- a/.ci/e2eTestingIngestManagerDaily.groovy +++ b/.ci/e2eTestingIngestManagerDaily.groovy @@ -45,7 +45,7 @@ pipeline { booleanParam(name: 'forceSkipGitChecks', value: true), booleanParam(name: 'forceSkipPresubmit', value: true), booleanParam(name: 'notifyOnGreenBuilds', value: true), - string(name: 'runTestsSuite', value: 'ingest-manager'), + string(name: 'runTestsSuites', value: 'ingest-manager'), string(name: 'SLACK_CHANNEL', value: "ingest-management"), ], propagate: false, diff --git a/.ci/e2eTestingIntegrationsDaily.groovy b/.ci/e2eTestingIntegrationsDaily.groovy index 50b80e5b68..3a8a7194c7 100644 --- a/.ci/e2eTestingIntegrationsDaily.groovy +++ b/.ci/e2eTestingIntegrationsDaily.groovy @@ -44,7 +44,7 @@ pipeline { parameters: [ booleanParam(name: 'forceSkipGitChecks', value: true), booleanParam(name: 'forceSkipPresubmit', value: true), - string(name: 'runTestsSuite', value: 'metricbeat'), + string(name: 'runTestsSuites', value: 'metricbeat'), string(name: 'SLACK_CHANNEL', value: "integrations"), ], propagate: false, diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh index e954b1c6d3..634b97b3be 100755 --- a/.ci/scripts/functional-test.sh +++ b/.ci/scripts/functional-test.sh @@ -11,13 +11,13 @@ set -euxo pipefail # # Parameters: # - SUITE - that's the suite to be tested. Default '' which means all of them. -# - FEATURE - that's the feature to be tested. Default '' which means all of them. +# - TAGS - that's the tags to be tested. Default '' which means all of them. # - STACK_VERSION - that's the version of the stack to be tested. Default '7.9.0'. # - METRICBEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.9.0'. # SUITE=${1:-''} -FEATURE=${2:-''} +TAGS=${2:-''} STACK_VERSION=${3:-'7.9.0'} METRICBEAT_VERSION=${4:-'7.9.0'} TARGET_OS=${GOOS:-linux} @@ -26,19 +26,16 @@ TARGET_ARCH=${GOARCH:-amd64} rm -rf outputs || true mkdir -p outputs -## Parse FEATURE if not ALL then enable the flags to be passed to the functional-test wrapper -REPORT='' -if [ "${FEATURE}" != "" ] && [ "${FEATURE}" != "all" ] ; then - REPORT=outputs/TEST-${SUITE}-${FEATURE} -else - FEATURE='' - REPORT=outputs/TEST-functional-tests +## Parse TAGS if not empty then enable the flags to be passed to the functional-test wrapper +REPORT=outputs/TEST-${SUITE} +if [ "${TAGS}" != "" ] ; then + REPORT=outputs/TEST-${SUITE}-${TAGS} fi ## Generate test report even if make failed. set +e exit_status=0 -if ! SUITE=${SUITE} FEATURE=${FEATURE} FORMAT=junit STACK_VERSION=${STACK_VERSION} METRICBEAT_VERSION=${METRICBEAT_VERSION} make --no-print-directory -C e2e functional-test | tee ${REPORT} ; then +if ! SUITE=${SUITE} TAGS="${TAGS}" FORMAT=junit STACK_VERSION=${STACK_VERSION} METRICBEAT_VERSION=${METRICBEAT_VERSION} make --no-print-directory -C e2e functional-test | tee ${REPORT} ; then echo 'ERROR: functional-test failed' exit_status=1 fi diff --git a/e2e/Makefile b/e2e/Makefile index 1ae83313d8..d09efbcb5d 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -1,5 +1,5 @@ SUITE?=metricbeat -FEATURE?= +TAGS?= DEVELOPER_MODE?=false FORMAT?=pretty LOG_INCLUDE_TIMESTAMP?=TRUE @@ -10,8 +10,8 @@ METRICBEAT_VERSION?= PICKLES_VERSION?="2.20.1" VERSION_VALUE=`cat ../cli/VERSION.txt` -ifneq ($(FEATURE),) -FEATURE_FLAG=--tags +ifneq ($(TAGS),) +TAGS_FLAG=--tags endif GO_IMAGE_TAG?='stretch' @@ -50,7 +50,7 @@ functional-test: install-godog METRICBEAT_VERSION=${METRICBEAT_VERSION} \ STACK_VERSION=${STACK_VERSION} \ DEVELOPER_MODE=${DEVELOPER_MODE} \ - godog --format=${FORMAT} ${FEATURE_FLAG} ${FEATURE} + godog --format=${FORMAT} ${TAGS_FLAG} ${TAGS} .PHONY: lint lint: diff --git a/e2e/runner_test.go b/e2e/runner_test.go index 6899a2a064..af3c4e2c92 100644 --- a/e2e/runner_test.go +++ b/e2e/runner_test.go @@ -87,7 +87,7 @@ func parseFeatureFlags(flags []string) ([]string, []*contextMetadata) { metadatas := []*contextMetadata{} featurePaths := []string{} - if len(flags) == 1 && flags[0] == "all" { + if len(flags) == 1 && flags[0] == "" { for k, metadata := range supportedProducts { metadata.name = k // match key with context name metadatas = append(metadatas, metadata)