CI: enable AWS cloud testing on a PR basis if changes in certain files

Jenkinsfile

@@ -291,13 +291,13 @@ def generateStages(Map args = [:]) {
 def cloud(Map args = [:]) {
   withGithubNotify(context: args.context) {
     withNode(labels: args.label, forceWorkspace: true){
-      startCloudTestEnv(name: args.directory, dirs: args.dirs)
+      startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS)
     }
-    withCloudTestEnv() {
+    withCloudTestEnv(args) {
       try {
         target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id)
       } finally {
-        terraformCleanup(name: args.directory, dir: args.directory)
+        terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS)
       }
     }
   }
@@ -868,14 +868,15 @@ def tarAndUploadArtifacts(Map args = [:]) {
 * This method executes a closure with credentials for cloud test
 * environments.
 */
-def withCloudTestEnv(Closure body) {
+def withCloudTestEnv(Map args = [:], Closure body) {
   def maskedVars = []
   def testTags = "${env.TEST_TAGS}"
 
   // Allow AWS credentials when the build was configured to do so with:
   //   - the cloudtests build parameters
   //   - the aws github label
-  if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws')) {
+  //   - forced with the cloud argument aws github label
+  if (params.allCloudTests || params.awsCloudTests || matchesPrLabel(label: 'aws') || args.get('withAWS', false)) {
     testTags = "${testTags},aws"
     def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data
     if (!aws.containsKey('access_key')) {
@@ -889,6 +890,7 @@ def withCloudTestEnv(Closure body) {
       [var: "AWS_ACCESS_KEY_ID", password: aws.access_key],
       [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key],
     ])
+    log(level: 'INFO', text: 'withCloudTestEnv: it has been configured to run in AWS.')
   }
 
   withEnv([
@@ -914,7 +916,7 @@ def startCloudTestEnv(Map args = [:]) {
   String name = normalise(args.name)
   def dirs = args.get('dirs',[])
   stage("${name}-prepare-cloud-env"){
-    withCloudTestEnv() {
+    withCloudTestEnv(args) {
       withBeatsEnv(archive: false, withModule: false) {
         try {
           dirs?.each { folder ->
@@ -957,7 +959,7 @@ def terraformCleanup(Map args = [:]) {
   String name = normalise(args.name)
   String directory = args.dir
   stage("${name}-tear-down-cloud-env"){
-    withCloudTestEnv() {
+    withCloudTestEnv(args) {
       withBeatsEnv(archive: false, withModule: false) {
         unstash("terraform-${name}")
         retryWithSleep(retries: 2, seconds: 5, backoff: true) {
@@ -1082,6 +1084,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
   public run(Map args = [:]){
     steps.stageStatusCache(args){
       def withModule = args.content.get('withModule', false)
+      def withAWS = args.content.get('withAWS', false)
       //
       // What's the retry policy for fighting the flakiness:
       //   1) Lint/Packaging/Cloud/k8sTest stages don't retry, since their failures are normally legitim
@@ -1140,7 +1143,7 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
         steps.k8sTest(context: args.context, versions: args.content.k8sTest.split(','), label: args.label, id: args.id)
       }
       if(args?.content?.containsKey('cloud')) {
-        steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id)
+        steps.cloud(context: args.context, command: args.content.cloud, directory: args.project, label: args.label, withModule: withModule, dirs: args.content.dirs, id: args.id, withAWS: withAWS)
       }
     }
   }

x-pack/filebeat/Jenkinsfile.yml

@@ -113,6 +113,20 @@ stages:
         labels:
           - "aws"
       stage: extended
+    cloudAWS:
+        cloud: "mage build test"
+        withAWS: true          ## Enable the tests to run in AWS
+        withModule: true       ## run the ITs only if the changeset affects a specific module.
+        dirs:                  ## run the cloud tests for the given modules.
+          - "x-pack/filebeat/input/awss3/_meta/terraform"
+        when:                  ## Override the top-level when.
+          changeset:           ## when PR contains any of those entries in the changeset
+            - "^x-pack/filebeat/input/awss3/.*"
+            - "^x-pack/filebeat/module/aws/.*"
+            - "^x-pack/filebeat/input/awscloudwatch/.*"
+            - "^x-pack/filebeat/Jenkinsfile.yml"
+            - "^x-pack/libbeat/common/aws/.*"
+        stage: extended
     #windows-7-32:  See https://github.com/elastic/beats/issues/22315
     #    mage: "mage build unitTest"
     #    platforms:             ## override default labels in this specific stage.

x-pack/metricbeat/Jenkinsfile.yml

@@ -45,6 +45,18 @@ stages:
             labels:
                 - "aws"
         stage: extended
+    cloudAWS:
+        cloud: "mage build test"
+        withAWS: true          ## Enable the tests to run in AWS
+        withModule: true       ## run the ITs only if the changeset affects a specific module.
+        dirs:                  ## run the cloud tests for the given modules.
+          - "x-pack/metricbeat/module/aws"
+        when:                  ## Override the top-level when.
+          changeset:         ## when PR contains any of those entries in the changeset
+            - "^x-pack/metricbeat/module/aws/.*"
+            - "^x-pack/metricbeat/Jenkinsfile.yml"
+            - "^x-pack/libbeat/common/aws/.*"
+        stage: extended
     macos:
         mage: "mage build unitTest"
         platforms:             ## override default label in this specific stage.