feat: support executing nightly tests on CI (#669) (#680)

* docs: document SKIP_SCENARIOS variable

* chore: support running nightly scenarios only from the nighltly builds

* fix: wrong variable assignment

.ci/Jenkinsfile

@@ -7,6 +7,7 @@ pipeline {
   environment {
     REPO = 'e2e-testing'
     BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    NIGHTLY_TAG="@nightly"
     NOTIFY_TO = credentials('notify-to')
     JOB_GCS_BUCKET = credentials('gcs-bucket')
     JOB_GIT_CREDENTIALS = "2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken"
@@ -28,6 +29,7 @@ pipeline {
   }
   parameters {
     booleanParam(name: "SKIP_SCENARIOS", defaultValue: true, description: "If it's needed to skip those scenarios marked as @skip. Default true")
+    booleanParam(name: "NIGHTLY_SCENARIOS", defaultValue: false, description: "If it's needed to include the scenarios marked as @nightly in the test execution. Default false")
     string(name: 'runTestsSuites', defaultValue: '', description: 'A comma-separated list of test suites to run (default: empty to run all test suites)')
     booleanParam(name: "forceSkipGitChecks", defaultValue: false, description: "If it's needed to check for Git changes to filter by modified sources")
     booleanParam(name: "forceSkipPresubmit", defaultValue: false, description: "If it's needed to execute the pre-submit tests: unit and precommit.")
@@ -59,6 +61,7 @@ pipeline {
         PATH = "${env.PATH}:${env.WORKSPACE}/bin:${env.WORKSPACE}/${env.BASE_DIR}/.ci/scripts"
         GO111MODULE = 'on'
         SKIP_SCENARIOS = "${params.SKIP_SCENARIOS}"
+        NIGHTLY_SCENARIOS = "${params.NIGHTLY_SCENARIOS}"
         SLACK_CHANNEL = "${params.SLACK_CHANNEL.trim()}"
         ELASTIC_AGENT_DOWNLOAD_URL = "${params.ELASTIC_AGENT_DOWNLOAD_URL.trim()}"
         ELASTIC_AGENT_VERSION = "${params.ELASTIC_AGENT_VERSION.trim()}"
@@ -303,6 +306,16 @@ def generateFunctionalTestStep(Map args = [:]){
   def sneakCaseSuite = suite.toUpperCase().replaceAll("-", "_")
   def stackVersion = env."${sneakCaseSuite}_STACK_VERSION"
   def tags = args.get('tags')
+
+  // We will decide whether to include the nightly tests in the execution at CI time, only.
+  // On the other hand, the developers can use the TAGS environment variable locally.
+  // Finally, we positively know that tags are not empty, so we can use AND operator.
+  def excludeNightlyTag = " && ~${NIGHTLY_TAG}"
+  if ("${NIGHTLY_SCENARIOS}" == "true") {
+    excludeNightlyTag = ""
+  }
+  tags += excludeNightlyTag
+
   return {
     node("${platform} && immutable && docker") {
       try {

.ci/e2eTestingFleetDaily.groovy

@@ -45,6 +45,7 @@ pipeline {
             booleanParam(name: 'forceSkipGitChecks', value: true),
             booleanParam(name: 'forceSkipPresubmit', value: true),
             booleanParam(name: 'notifyOnGreenBuilds', value: true),
+            booleanParam(name: 'NIGHTLY_SCENARIOS', value: true),
             string(name: 'runTestsSuites', value: 'fleet'),
             string(name: 'SLACK_CHANNEL', value: "ingest-management"),
           ],

e2e/README.md

@@ -134,6 +134,7 @@ We are going to enumerate the variables that will affect the product versions us
 ### Environment variables affecting the build
 The following environment variables affect how the tests are run in both the CI and a local machine.
 
+- `SKIP_SCENARIOS`: Set this environment variable to `false` if it's needed to include the scenarios annotated as `@skip` in the current test execution. Default value: `true`.
 - `BEATS_USE_CI_SNAPSHOTS`: Set this environment variable to `true` if it's needed to use the binary snapshots produced by Beats CI instead of the official releases. The snapshots will be downloaded from a bucket in Google Cloud Storage. Default: `false`. This variable is used by the Beats repository, when testing the artifacts generated by the packaging job.
 - `LOG_LEVEL`: Set this environment variable to `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR` or `FATAL` to set the log level in the project. Default: `INFO`.
 - `DEVELOPER_MODE`: Set this environment variable to `true` to activate developer mode, which means not destroying the services provisioned by the test framework. Default: `false`.