From aa475907dc08c284466aad1899ab3fbf6a957b91 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Wed, 27 Apr 2022 16:20:48 +0100 Subject: [PATCH] [ci][terraform] tags with metadata (#31355) --- Jenkinsfile | 121 ++++++++++-------- .../input/awss3/_meta/terraform/main.tf | 9 ++ .../input/awss3/_meta/terraform/variables.tf | 23 ++++ x-pack/metricbeat/module/aws/terraform.tf | 11 +- x-pack/metricbeat/module/aws/variables.tf | 22 ++++ 5 files changed, 129 insertions(+), 57 deletions(-) create mode 100644 x-pack/metricbeat/module/aws/variables.tf diff --git a/Jenkinsfile b/Jenkinsfile index 503cd4f6c21..3d00e505848 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -303,13 +303,13 @@ def generateStages(Map args = [:]) { def cloud(Map args = [:]) { withGithubNotify(context: args.context) { withNode(labels: args.label, forceWorkspace: true){ - startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS) - } - withCloudTestEnv(args) { - try { - target(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) - } finally { - terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS) + withCloudTestEnv(args) { + startCloudTestEnv(name: args.directory, dirs: args.dirs, withAWS: args.withAWS) + try { + targetWithoutNode(context: args.context, command: args.command, directory: args.directory, label: args.label, withModule: args.withModule, isMage: true, id: args.id) + } finally { + terraformCleanup(name: args.directory, dir: args.directory, withAWS: args.withAWS) + } } } } @@ -555,12 +555,21 @@ def e2e_with_entrypoint(Map args = [:]) { } } +/** +* This method runs in a node +*/ +def target(Map args = [:]) { + withNode(labels: args.label, forceWorkspace: true){ + targetWithoutNode(args) + } +} + /** * This method runs the given command supporting two kind of scenarios: * - make -C then the dir(location) is not required, aka by disaling isMage: false * - mage then the dir(location) is required, aka by enabling isMage: true. */ -def target(Map args = [:]) { +def targetWithoutNode(Map args = [:]) { def command = args.command def context = args.context def directory = args.get('directory', '') @@ -572,34 +581,32 @@ def target(Map args = [:]) { def dockerArch = args.get('dockerArch', 'amd64') def enableRetry = args.get('enableRetry', false) def withGCP = args.get('withGCP', false) - withNode(labels: args.label, forceWorkspace: true){ - withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { - dumpVariables() - withTools(k8s: installK8s, gcp: withGCP) { - // make commands use -C while mage commands require the dir(folder) - // let's support this scenario with the location variable. - dir(isMage ? directory : '') { - if (enableRetry) { - // Retry the same command to bypass any kind of flakiness. - // Downside: genuine failures will be repeated. - retry(3) { - cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}") - } - } else { + withGithubNotify(context: "${context}") { + withBeatsEnv(archive: true, withModule: withModule, directory: directory, id: args.id) { + dumpVariables() + withTools(k8s: installK8s, gcp: withGCP) { + // make commands use -C while mage commands require the dir(folder) + // let's support this scenario with the location variable. + dir(isMage ? directory : '') { + if (enableRetry) { + // Retry the same command to bypass any kind of flakiness. + // Downside: genuine failures will be repeated. + retry(3) { cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}") } + } else { + cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}") } } - // Publish packages should happen always to easily consume those artifacts if the - // e2e were triggered and failed. - if (isPackaging) { - publishPackages("${directory}") - pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch) - } - if(isE2E) { - e2e(args) - } + } + // Publish packages should happen always to easily consume those artifacts if the + // e2e were triggered and failed. + if (isPackaging) { + publishPackages("${directory}") + pushCIDockerImages(beatsFolder: "${directory}", arch: dockerArch) + } + if(isE2E) { + e2e(args) } } } @@ -900,26 +907,24 @@ def startCloudTestEnv(Map args = [:]) { String name = normalise(args.name) def dirs = args.get('dirs',[]) stage("${name}-prepare-cloud-env"){ - withCloudTestEnv(args) { - withBeatsEnv(archive: false, withModule: false) { - try { - dirs?.each { folder -> - retryWithSleep(retries: 2, seconds: 5, backoff: true){ - terraformApply(folder) - } - } - } catch(err) { - dirs?.each { folder -> - // If it failed then cleanup without failing the build - sh(label: 'Terraform Cleanup', script: ".ci/scripts/terraform-cleanup.sh ${folder}", returnStatus: true) + withBeatsEnv(archive: false, withModule: false) { + try { + dirs?.each { folder -> + retryWithSleep(retries: 2, seconds: 5, backoff: true){ + terraformApply(folder) } - error('startCloudTestEnv: terraform apply failed.') - } finally { - // Archive terraform states in case manual cleanup is needed. - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') } - stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') + } catch(err) { + dirs?.each { folder -> + // If it failed then cleanup without failing the build + sh(label: 'Terraform Cleanup', script: ".ci/scripts/terraform-cleanup.sh ${folder}", returnStatus: true) + } + error('startCloudTestEnv: terraform apply failed.') + } finally { + // Archive terraform states in case manual cleanup is needed. + archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') } + stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') } } } @@ -930,7 +935,13 @@ def startCloudTestEnv(Map args = [:]) { def terraformApply(String directory) { terraformInit(directory) dir(directory) { - sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + withEnv(["TF_VAR_BRANCH=${env.BRANCH_NAME.toLowerCase().replaceAll('[^a-z0-9-]', '-')}", + "TF_VAR_BUILD_ID=${BUILD_ID}", + "TF_VAR_CREATED_DATE=${new Date().getTime()}", + "TF_VAR_ENVIRONMENT=ci", + "TF_VAR_REPO=${env.REPO}"]) { + sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") + } } } @@ -943,12 +954,10 @@ def terraformCleanup(Map args = [:]) { String name = normalise(args.name) String directory = args.dir stage("${name}-tear-down-cloud-env"){ - withCloudTestEnv(args) { - withBeatsEnv(archive: false, withModule: false) { - unstash("terraform-${name}") - retryWithSleep(retries: 2, seconds: 5, backoff: true) { - sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") - } + withBeatsEnv(archive: false, withModule: false) { + unstash("terraform-${name}") + retryWithSleep(retries: 2, seconds: 5, backoff: true) { + sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") } } } diff --git a/x-pack/filebeat/input/awss3/_meta/terraform/main.tf b/x-pack/filebeat/input/awss3/_meta/terraform/main.tf index 62e86abc787..81a24b3f161 100644 --- a/x-pack/filebeat/input/awss3/_meta/terraform/main.tf +++ b/x-pack/filebeat/input/awss3/_meta/terraform/main.tf @@ -9,6 +9,15 @@ terraform { provider "aws" { region = var.aws_region + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } } resource "random_string" "random" { diff --git a/x-pack/filebeat/input/awss3/_meta/terraform/variables.tf b/x-pack/filebeat/input/awss3/_meta/terraform/variables.tf index 2c4fb00786b..78b0a474147 100644 --- a/x-pack/filebeat/input/awss3/_meta/terraform/variables.tf +++ b/x-pack/filebeat/input/awss3/_meta/terraform/variables.tf @@ -3,3 +3,26 @@ variable "aws_region" { type = string default = "us-east-1" } + +variable "BRANCH" { + description = "Branch name or pull request for tagging purposes" + default = "unknown-branch" +} + +variable "BUILD_ID" { + description = "Build ID in the CI for tagging purposes" + default = "unknown-build" +} + +variable "CREATED_DATE" { + description = "Creation date in epoch time for tagging purposes" + default = "unknown-date" +} + +variable "ENVIRONMENT" { + default = "unknown-environment" +} + +variable "REPO" { + default = "unknown-repo-name" +} diff --git a/x-pack/metricbeat/module/aws/terraform.tf b/x-pack/metricbeat/module/aws/terraform.tf index 93c85836fc2..6a7ec6e2478 100644 --- a/x-pack/metricbeat/module/aws/terraform.tf +++ b/x-pack/metricbeat/module/aws/terraform.tf @@ -1,5 +1,14 @@ provider "aws" { - version = "~> 2.58" + version = "~> 3.33" + default_tags { + tags = { + environment = var.ENVIRONMENT + repo = var.REPO + branch = var.BRANCH + build = var.BUILD_ID + created_date = var.CREATED_DATE + } + } } provider "random" { diff --git a/x-pack/metricbeat/module/aws/variables.tf b/x-pack/metricbeat/module/aws/variables.tf new file mode 100644 index 00000000000..e4b95d471b2 --- /dev/null +++ b/x-pack/metricbeat/module/aws/variables.tf @@ -0,0 +1,22 @@ +variable "BRANCH" { + description = "Branch name or pull request for tagging purposes" + default = "unknown-branch" +} + +variable "BUILD_ID" { + description = "Build ID in the CI for tagging purposes" + default = "unknown-build" +} + +variable "CREATED_DATE" { + description = "Creation date in epoch time for tagging purposes" + default = "unknown-date" +} + +variable "ENVIRONMENT" { + default = "unknown-environment" +} + +variable "REPO" { + default = "unknown-repo-name" +}