feat: support a comma-separated list of test suites (#324) (#329)

* chore: rename variable to represent godog tags * fix: double quote variable * chore: rename jenkins parameter * chore: use empty instead of all as value for the tags to be run * chore: simplify variable value * chore: rename variable * feat: support passing a comma-separated list of test suites to be run * chore: extract common code to a method * chore: rename variable * chore: trim test suite in the list * chore: add logs * fix: forgot "each" for the array * chore: extract slack send to a function * chore: format slack message * chore: format message for empty test suites * chore: rename variable to avoid conflicts with Jenkins params * chore: simplify loop Co-authored-by: Victor Martinez <[email protected]> * chore: simplify variables inside loop Co-authored-by: Victor Martinez <[email protected]> # Conflicts: # .ci/scripts/functional-test.sh
elastic · Sep 28, 2020 · 2c44ba2 · 2c44ba2
1 parent d430ecb
commit 2c44ba2
Show file tree

Hide file tree

Showing 8 changed files with 77 additions and 56 deletions.
diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml
@@ -1,24 +1,24 @@
 ---
 SUITES:
   - suite: "helm"
-    feature: "apm-server"
+    tags: "apm-server"
   - suite: "helm"
-    feature: "filebeat"
+    tags: "filebeat"
   - suite: "helm"
-    feature: "metricbeat"
+    tags: "metricbeat"
   - suite: "ingest-manager"
-    feature: "agent_endpoint_integration"
+    tags: "agent_endpoint_integration"
   - suite: "ingest-manager"
-    feature: "stand_alone_mode"
+    tags: "stand_alone_mode"
   - suite: "ingest-manager"
-    feature: "fleet_mode"
+    tags: "fleet_mode"
   - suite: "metricbeat"
-    feature: "apache"
+    tags: "apache"
   - suite: "metricbeat"
-    feature: "metricbeat"
+    tags: "metricbeat"
   - suite: "metricbeat"
-    feature: "mysql"
+    tags: "mysql"
   - suite: "metricbeat"
-    feature: "redis"
+    tags: "redis"
   - suite: "metricbeat"
-    feature: "vsphere"
+    tags: "vsphere"
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -27,7 +27,7 @@ pipeline {
     issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*')
   }
   parameters {
-    choice(name: 'runTestsSuite', choices: ['all', 'helm', 'ingest-manager', 'metricbeat'], description: 'Choose which test suite to run (default: all)')
+    string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)')
     booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
     booleanParam(name: "forceSkipPresubmit", defaultValue: false, description: "If it's needed to execute the pre-submit tests: unit and precommit.")
     booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify with green builds.")
@@ -184,19 +184,20 @@ pipeline {
               unstash 'source'
               dir("${BASE_DIR}") {
                 script {
-                  def suiteParam = params.runTestsSuite
-                  def suites = readYaml(file: '.ci/.e2e-tests.yaml')
+                  def suitesParam = params.runTestsSuites
+                  def existingSuites = readYaml(file: '.ci/.e2e-tests.yaml')
                   def parallelTasks = [:]
-                  suites['SUITES'].each { item ->
-                    def suite = item.suite
-                    def feature = item.feature
-                    if (suiteParam == "all" || suiteParam == suite) {
-                      def regexps = [ "^e2e/_suites/${suite}/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go" ]
-                      if ("${FORCE_SKIP_GIT_CHECKS}" == "true" || isGitRegionMatch(patterns: regexps, shouldMatchAll: false)) {
-                        log(level: 'INFO', text: "Adding ${suite}:${feature} test suite to the build execution")
-                        parallelTasks["${suite}_${feature}"] = generateFunctionalTestStep(suite: "${suite}", feature: "${feature}")
-                      } else {
-                        log(level: 'WARN', text: "The ${suite}:${feature} test suite won't be executed because there are no modified files")
+
+                  if (suitesParam == "") {
+                    log(level: 'DEBUG', text: "Iterate through existing test suites")
+                    existingSuites['SUITES'].each { item ->
+                      checkTestSuite(parallelTasks, item.suite, item.tags)
+                    }
+                  } else {
+                    log(level: 'DEBUG', text: "Iterate through the comma-separated test suites (${suitesParam}), comparing with the existing test suites")
+                    suitesParam.split(',').each { suiteParam ->
+                      existingSuites['SUITES'].findAll { suiteParam.trim() == it.suite }.each { item ->
+                          checkTestSuite(parallelTasks, item.suite, item.tags)
                       }
                     }
                   }
@@ -236,24 +237,47 @@ pipeline {
     }
     success {
       whenTrue(!isPR() && params.notifyOnGreenBuilds) {
-        slackSend(channel: "#${env.SLACK_CHANNEL}", color: 'good',
-                  message: "[${params.runTestsSuite}] Build Passed: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.RUN_DISPLAY_URL}|Open>).",
-                  tokenCredentialId: 'jenkins-slack-integration-token')
+        doSlackSend('good')
       }
     }
     unsuccessful {
       whenFalse(isPR()) {
-        slackSend(channel: "#${env.SLACK_CHANNEL}", color: 'danger',
-                  message: "[${params.runTestsSuite}] Build Failed: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.RUN_DISPLAY_URL}|Open>).",
-                  tokenCredentialId: 'jenkins-slack-integration-token')
+        doSlackSend('danger')
       }
     }
   }
 }
 
-def generateStep(Map params = [:]){
-  def oss = params.get('oss')
-  def platform = params.get('platform')
+def checkTestSuite(Map parallelTasks = [:], String suite, String tags) {
+  def regexps = [ "^e2e/_suites/${suite}/.*", "^.ci/.*", "^cli/.*", "^e2e/.*\\.go" ]
+  if ("${FORCE_SKIP_GIT_CHECKS}" == "true" || isGitRegionMatch(patterns: regexps, shouldMatchAll: false)) {
+    log(level: 'INFO', text: "Adding ${suite}:${tags} test suite to the build execution")
+    parallelTasks["${suite}_${tags}"] = generateFunctionalTestStep(suite: "${suite}", tags: "${tags}")
+  } else {
+    log(level: 'WARN', text: "The ${suite}:${tags} test suite won't be executed because there are no modified files")
+  }
+}
+
+def doSlackSend(String color) {
+  def buildStatus = "Failed"
+  if (color == "good") {
+    buildStatus = "Passed"
+  }
+
+  def testSuites = "${params.runTestsSuites}"
+  if (testSuites == "") {
+    testsSuites = "All suites"
+  }
+
+  def message = "*[${testsSuites}]* Build `${buildStatus}`: ${env.JOB_NAME} ${env.BUILD_NUMBER} (<${env.RUN_DISPLAY_URL}|Open>)."
+
+  slackSend(channel: "#${env.SLACK_CHANNEL}", color: "${color}", message: "${message}",tokenCredentialId: 'jenkins-slack-integration-token')
+}
+
+
+def generateStep(Map args = [:]){
+  def oss = args.get('oss')
+  def platform = args.get('platform')
   return {
     node('ubuntu-18.04 && immutable && docker') {
       try {
@@ -273,11 +297,11 @@ def generateStep(Map params = [:]){
   }
 }
 
-def generateFunctionalTestStep(Map params = [:]){
-  def suite = params.get('suite')
+def generateFunctionalTestStep(Map args = [:]){
+  def suite = args.get('suite')
   def sneakCaseSuite = suite.toUpperCase().replaceAll("-", "_")
   def stackVersion = env."${sneakCaseSuite}_STACK_VERSION"
-  def feature = params.get('feature')
+  def tags = args.get('tags')
   return {
     node('ubuntu-18.04 && immutable && docker') {
       try {
@@ -289,11 +313,11 @@ def generateFunctionalTestStep(Map params = [:]){
           }
           retry(3){
             dir("${BASE_DIR}"){
-              sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${feature}"
+              sh script: """.ci/scripts/install-test-dependencies.sh "${suite}" """, label: "Install test dependencies for ${suite}:${tags}"
             }
           }
           dir("${BASE_DIR}"){
-            sh script: """.ci/scripts/functional-test.sh "${suite}" "${feature}" "${stackVersion}" "${METRICBEAT_VERSION}" """, label: "Run functional tests for ${suite}:${feature}"
+            sh script: """.ci/scripts/functional-test.sh "${suite}" "${tags}" "${stackVersion}" "${METRICBEAT_VERSION}" """, label: "Run functional tests for ${suite}:${tags}"
           }
         }
       } catch(e) {

diff --git a/.ci/e2eTestingHelmDaily.groovy b/.ci/e2eTestingHelmDaily.groovy
@@ -44,7 +44,7 @@ pipeline {
           parameters: [
             booleanParam(name: 'forceSkipGitChecks', value: true),
             booleanParam(name: 'forceSkipPresubmit', value: true),
-            string(name: 'runTestsSuite', value: 'helm'),
+            string(name: 'runTestsSuites', value: 'helm'),
             string(name: 'SLACK_CHANNEL', value: "integrations"),
           ],
           propagate: false,

diff --git a/.ci/e2eTestingIngestManagerDaily.groovy b/.ci/e2eTestingIngestManagerDaily.groovy
@@ -45,7 +45,7 @@ pipeline {
             booleanParam(name: 'forceSkipGitChecks', value: true),
             booleanParam(name: 'forceSkipPresubmit', value: true),
             booleanParam(name: 'notifyOnGreenBuilds', value: true),
-            string(name: 'runTestsSuite', value: 'ingest-manager'),
+            string(name: 'runTestsSuites', value: 'ingest-manager'),
             string(name: 'SLACK_CHANNEL', value: "ingest-management"),
           ],
           propagate: false,

diff --git a/.ci/e2eTestingIntegrationsDaily.groovy b/.ci/e2eTestingIntegrationsDaily.groovy
@@ -44,7 +44,7 @@ pipeline {
           parameters: [
             booleanParam(name: 'forceSkipGitChecks', value: true),
             booleanParam(name: 'forceSkipPresubmit', value: true),
-            string(name: 'runTestsSuite', value: 'metricbeat'),
+            string(name: 'runTestsSuites', value: 'metricbeat'),
             string(name: 'SLACK_CHANNEL', value: "integrations"),
           ],
           propagate: false,

diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh
@@ -11,13 +11,13 @@ set -euxo pipefail
 #
 # Parameters:
 #   - SUITE - that's the suite to be tested. Default '' which means all of them.
-#   - FEATURE - that's the feature to be tested. Default '' which means all of them.
+#   - TAGS - that's the tags to be tested. Default '' which means all of them.
 #   - STACK_VERSION - that's the version of the stack to be tested. Default '7.9.0'.
 #   - METRICBEAT_VERSION - that's the version of the metricbeat to be tested. Default '7.9.0'.
 #
 
 SUITE=${1:-''}
-FEATURE=${2:-''}
+TAGS=${2:-''}
 STACK_VERSION=${3:-'7.9.0'}
 METRICBEAT_VERSION=${4:-'7.9.0'}
 TARGET_OS=${GOOS:-linux}
@@ -26,19 +26,16 @@ TARGET_ARCH=${GOARCH:-amd64}
 rm -rf outputs || true
 mkdir -p outputs
 
-## Parse FEATURE if not ALL then enable the flags to be passed to the functional-test wrapper
-REPORT=''
-if [ "${FEATURE}" != "" ] && [ "${FEATURE}" != "all" ] ; then
-  REPORT=outputs/TEST-${SUITE}-${FEATURE}
-else
-  FEATURE=''
-  REPORT=outputs/TEST-functional-tests
+## Parse TAGS if not empty then enable the flags to be passed to the functional-test wrapper
+REPORT=outputs/TEST-${SUITE}
+if [ "${TAGS}" != "" ] ; then
+  REPORT=outputs/TEST-${SUITE}-${TAGS}
 fi
 
 ## Generate test report even if make failed.
 set +e
 exit_status=0
-if ! SUITE=${SUITE} FEATURE=${FEATURE} FORMAT=junit STACK_VERSION=${STACK_VERSION} METRICBEAT_VERSION=${METRICBEAT_VERSION} make --no-print-directory -C e2e functional-test | tee ${REPORT}  ; then
+if ! SUITE=${SUITE} TAGS="${TAGS}" FORMAT=junit STACK_VERSION=${STACK_VERSION} METRICBEAT_VERSION=${METRICBEAT_VERSION} make --no-print-directory -C e2e functional-test | tee ${REPORT}  ; then
   echo 'ERROR: functional-test failed'
   exit_status=1
 fi

diff --git a/e2e/Makefile b/e2e/Makefile
@@ -1,5 +1,5 @@
 SUITE?=metricbeat
-FEATURE?=
+TAGS?=
 DEVELOPER_MODE?=false
 FORMAT?=pretty
 LOG_INCLUDE_TIMESTAMP?=TRUE
@@ -10,8 +10,8 @@ METRICBEAT_VERSION?=
 PICKLES_VERSION?="2.20.1"
 VERSION_VALUE=`cat ../cli/VERSION.txt`
 
-ifneq ($(FEATURE),)
-FEATURE_FLAG=--tags
+ifneq ($(TAGS),)
+TAGS_FLAG=--tags
 endif
 
 GO_IMAGE_TAG?='stretch'
@@ -50,7 +50,7 @@ functional-test: install-godog
 	METRICBEAT_VERSION=${METRICBEAT_VERSION} \
 	STACK_VERSION=${STACK_VERSION} \
 	DEVELOPER_MODE=${DEVELOPER_MODE} \
-	godog --format=${FORMAT} ${FEATURE_FLAG} ${FEATURE}
+	godog --format=${FORMAT} ${TAGS_FLAG} ${TAGS}
 
 .PHONY: lint
 lint:

diff --git a/e2e/runner_test.go b/e2e/runner_test.go
@@ -87,7 +87,7 @@ func parseFeatureFlags(flags []string) ([]string, []*contextMetadata) {
 	metadatas := []*contextMetadata{}
 	featurePaths := []string{}
 
-	if len(flags) == 1 && flags[0] == "all" {
+	if len(flags) == 1 && flags[0] == "" {
 		for k, metadata := range supportedProducts {
 			metadata.name = k // match key with context name
 			metadatas = append(metadatas, metadata)