[ci][terraform] tags with metadata (#31355)

elastic · Apr 27, 2022 · aa47590 · aa47590
1 parent 755cb8a
commit aa47590
Show file tree

Hide file tree

Showing 5 changed files with 129 additions and 57 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -303,13 +303,13 @@ def generateStages(Map args = [:]) {
 def cloud(Map args = [:]) {
   withGithubNotify(context: args.context) {
     withNode(labels: args.label, forceWorkspace: true){
-      startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS)
-    }
-    withCloudTestEnv(args) {
-      try {
-        target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id)
-      } finally {
-        terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS)
+      withCloudTestEnv(args) {
+        startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS)
+        try {
+          targetWithoutNode(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id)
+        } finally {
+          terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS)
+        }
       }
     }
   }
@@ -555,12 +555,21 @@ def e2e_with_entrypoint(Map args = [:]) {
   }
 }
 
+/**
+* This method runs in a node
+*/
+def target(Map args = [:]) {
+  withNode(labels: args.label, forceWorkspace: true){
+    targetWithoutNode(args)
+  }
+}
+
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false
 *  - mage then the dir(location) is required, aka by enabling isMage: true.
 */
-def target(Map args = [:]) {
+def targetWithoutNode(Map args = [:]) {
   def command = args.command
   def context = args.context
   def directory = args.get('directory', '')
@@ -572,34 +581,32 @@ def target(Map args = [:]) {
   def dockerArch = args.get('dockerArch', 'amd64')
   def enableRetry = args.get('enableRetry', false)
   def withGCP = args.get('withGCP', false)
-  withNode(labels: args.label, forceWorkspace: true){
-    withGithubNotify(context: "${context}") {
-      withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
-        dumpVariables()
-        withTools(k8s: installK8s, gcp: withGCP) {
-          // make commands use -C <folder> while mage commands require the dir(folder)
-          // let's support this scenario with the location variable.
-          dir(isMage ? directory : '') {
-            if (enableRetry) {
-              // Retry the same command to bypass any kind of flakiness.
-              // Downside: genuine failures will be repeated.
-              retry(3) {
-                cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
-              }
-            } else {
+  withGithubNotify(context: "${context}") {
+    withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) {
+      dumpVariables()
+      withTools(k8s: installK8s, gcp: withGCP) {
+        // make commands use -C <folder> while mage commands require the dir(folder)
+        // let's support this scenario with the location variable.
+        dir(isMage ? directory : '') {
+          if (enableRetry) {
+            // Retry the same command to bypass any kind of flakiness.
+            // Downside: genuine failures will be repeated.
+            retry(3) {
               cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
             }
+          } else {
+            cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
           }
         }
-        // Publish packages should happen always to easily consume those artifacts if the
-        // e2e were triggered and failed.
-        if (isPackaging) {
-          publishPackages("${directory}")
-          pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
-        }
-        if(isE2E) {
-          e2e(args)
-        }
+      }
+      // Publish packages should happen always to easily consume those artifacts if the
+      // e2e were triggered and failed.
+      if (isPackaging) {
+        publishPackages("${directory}")
+        pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch)
+      }
+      if(isE2E) {
+        e2e(args)
       }
     }
   }
@@ -900,26 +907,24 @@ def startCloudTestEnv(Map args = [:]) {
   String name = normalise(args.name)
   def dirs = args.get('dirs',[])
   stage("${name}-prepare-cloud-env"){
-    withCloudTestEnv(args) {
-      withBeatsEnv(archive: false, withModule: false) {
-        try {
-          dirs?.each { folder ->
-            retryWithSleep(retries: 2, seconds: 5, backoff: true){
-              terraformApply(folder)
-            }
-          }
-        } catch(err) {
-          dirs?.each { folder ->
-            // If it failed then cleanup without failing the build
-            sh(label: 'Terraform Cleanup', script: ".ci/scripts/terraform-cleanup.sh ${folder}", returnStatus: true)
+    withBeatsEnv(archive: false, withModule: false) {
+      try {
+        dirs?.each { folder ->
+          retryWithSleep(retries: 2, seconds: 5, backoff: true){
+            terraformApply(folder)
           }
-          error('startCloudTestEnv: terraform apply failed.')
-        } finally {
-          // Archive terraform states in case manual cleanup is needed.
-          archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
         }
-        stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
+      } catch(err) {
+        dirs?.each { folder ->
+          // If it failed then cleanup without failing the build
+          sh(label: 'Terraform Cleanup', script: ".ci/scripts/terraform-cleanup.sh ${folder}", returnStatus: true)
+        }
+        error('startCloudTestEnv: terraform apply failed.')
+      } finally {
+        // Archive terraform states in case manual cleanup is needed.
+        archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
       }
+      stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
     }
   }
 }
@@ -930,7 +935,13 @@ def startCloudTestEnv(Map args = [:]) {
 def terraformApply(String directory) {
   terraformInit(directory)
   dir(directory) {
-    sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve")
+    withEnv(["TF_VAR_BRANCH=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')}",
+             "TF_VAR_BUILD_ID=${BUILD_ID}",
+             "TF_VAR_CREATED_DATE=${new Date().getTime()}",
+             "TF_VAR_ENVIRONMENT=ci",
+             "TF_VAR_REPO=${env.REPO}"]) {
+      sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve")
+    }
   }
 }
 
@@ -943,12 +954,10 @@ def terraformCleanup(Map args = [:]) {
   String name = normalise(args.name)
   String directory = args.dir
   stage("${name}-tear-down-cloud-env"){
-    withCloudTestEnv(args) {
-      withBeatsEnv(archive: false, withModule: false) {
-        unstash("terraform-${name}")
-        retryWithSleep(retries: 2, seconds: 5, backoff: true) {
-          sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
-        }
+    withBeatsEnv(archive: false, withModule: false) {
+      unstash("terraform-${name}")
+      retryWithSleep(retries: 2, seconds: 5, backoff: true) {
+        sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
       }
     }
   }

diff --git a/x-pack/filebeat/input/awss3/_meta/terraform/main.tf b/x-pack/filebeat/input/awss3/_meta/terraform/main.tf
@@ -9,6 +9,15 @@ terraform {
 
 provider "aws" {
   region = var.aws_region
+  default_tags {
+    tags = {
+      environment  = var.ENVIRONMENT
+      repo         = var.REPO
+      branch       = var.BRANCH
+      build        = var.BUILD_ID
+      created_date = var.CREATED_DATE
+    }
+  }
 }
 
 resource "random_string" "random" {

diff --git a/x-pack/filebeat/input/awss3/_meta/terraform/variables.tf b/x-pack/filebeat/input/awss3/_meta/terraform/variables.tf
@@ -3,3 +3,26 @@ variable "aws_region" {
   type        = string
   default     = "us-east-1"
 }
+
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default = "unknown-branch"
+}
+
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default = "unknown-build"
+}
+
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default = "unknown-date"
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo-name"
+}
diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf
@@ -1,5 +1,14 @@
 provider "aws" {
-  version = "~> 2.58"
+  version = "~> 3.33"
+  default_tags {
+    tags = {
+      environment  = var.ENVIRONMENT
+      repo         = var.REPO
+      branch       = var.BRANCH
+      build        = var.BUILD_ID
+      created_date = var.CREATED_DATE
+    }
+  }
 }
 
 provider "random" {

diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf
@@ -0,0 +1,22 @@
+variable "BRANCH" {
+  description = "Branch name or pull request for tagging purposes"
+  default = "unknown-branch"
+}
+
+variable "BUILD_ID" {
+  description = "Build ID in the CI for tagging purposes"
+  default = "unknown-build"
+}
+
+variable "CREATED_DATE" {
+  description = "Creation date in epoch time for tagging purposes"
+  default = "unknown-date"
+}
+
+variable "ENVIRONMENT" {
+  default = "unknown-environment"
+}
+
+variable "REPO" {
+  default = "unknown-repo-name"
+}