From f9ecd71b5f3965fe2d204338d28759a1876c5ccb Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 21:47:11 +0000 Subject: [PATCH 01/14] refactor: move pipeline stages out into template --- terraform/pipeline/azure-pipelines.yml | 149 +------------------------ terraform/pipeline/deploy.yml | 149 +++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 148 deletions(-) create mode 100644 terraform/pipeline/deploy.yml diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index 16b7aa3c..dc04ced3 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -25,151 +25,4 @@ pool: vmImage: ubuntu-latest stages: - - stage: TerraformPlan - jobs: - - job: Plan - variables: - - name: OTHER_SOURCE - value: $[variables['System.PullRequest.SourceBranch']] - - name: INDIVIDUAL_SOURCE - value: $[variables['Build.SourceBranchName']] - - name: IS_TAG - value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] - - name: TARGET - value: $[variables['System.PullRequest.TargetBranch']] - steps: - # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python - # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable - - bash: | - WORKSPACE=$(python terraform/pipeline/workspace.py) - echo "##vso[task.setvariable variable=workspace;isOutput=true]$WORKSPACE" - - TAG_TYPE=$(python terraform/pipeline/tag.py) - echo "##vso[task.setvariable variable=tag_type;isOutput=true]$TAG_TYPE" - name: setvars - displayName: Determine deployment environment - env: - REASON: $(Build.Reason) - # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformInstaller#readme - - task: TerraformInstaller@0 - displayName: Install Terraform - inputs: - terraformVersion: 1.3.1 - # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme - - task: TerraformTaskV3@3 - displayName: Terraform init - inputs: - provider: azurerm - command: init - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false - # service connection - backendServiceArm: deployer - # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform - backendAzureRmContainerName: tfstate - backendAzureRmKey: terraform.tfstate - - task: TerraformTaskV3@3 - displayName: Select environment - inputs: - provider: azurerm - command: custom - customCommand: workspace - commandOptions: select $(setvars.workspace) - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer - - task: TerraformTaskV3@3 - displayName: Terraform plan - inputs: - provider: azurerm - command: plan - # wait for lock to be released, in case being used by another pipeline run - # https://discuss.hashicorp.com/t/terraform-plan-wait-for-lock-to-be-released/6870/2 - commandOptions: -input=false -lock-timeout=5m -out=$(Build.ArtifactStagingDirectory)/tfplan - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer - # need to publish the tfplan to used by next stage if it's going to run - - publish: "$(Build.ArtifactStagingDirectory)" - displayName: "Publish tfplan for use in TerraformApply" - artifact: savedPlan - condition: | - or( - in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), - eq(variables['setvars.tag_type'], 'test'), - eq(variables['setvars.tag_type'], 'prod') - ) - - stage: TerraformApply - dependsOn: TerraformPlan - variables: - - name: workspace - value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] - - name: tag_type - value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] - # only run on dev, test, or prod branches OR if it's a tag for test or prod - condition: | - or( - in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), - eq(variables['tag_type'], 'test'), - eq(variables['tag_type'], 'prod') - ) - jobs: - - deployment: Apply - condition: succeeded() - environment: Approval - variables: - - name: workspace - value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.workspace'] ] - - name: tag_type - value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.tag_type'] ] - strategy: - runOnce: - deploy: - steps: - - checkout: self - - download: current - displayName: "Download plan file published from TerraformPlan" - artifact: savedPlan - - task: TerraformInstaller@0 - displayName: Install Terraform - inputs: - terraformVersion: 1.3.1 - # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme - - task: TerraformTaskV3@3 - displayName: Terraform init - inputs: - provider: azurerm - command: init - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false - # service connection - backendServiceArm: deployer - # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform - backendAzureRmContainerName: tfstate - backendAzureRmKey: terraform.tfstate - - task: TerraformTaskV3@3 - displayName: Select environment - inputs: - provider: azurerm - command: custom - customCommand: workspace - commandOptions: select $(workspace) - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer - - task: TerraformTaskV3@3 - displayName: Terraform apply - inputs: - provider: azurerm - command: apply - # (ditto the lock comment above) - commandOptions: -input=false -lock-timeout=5m $(Pipeline.Workspace)/savedPlan/tfplan - workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" - # service connection - environmentServiceNameAzureRM: deployer + - template: deploy.yml diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml new file mode 100644 index 00000000..f4b4ece8 --- /dev/null +++ b/terraform/pipeline/deploy.yml @@ -0,0 +1,149 @@ +stages: + - stage: TerraformPlan + jobs: + - job: Plan + variables: + - name: OTHER_SOURCE + value: $[variables['System.PullRequest.SourceBranch']] + - name: INDIVIDUAL_SOURCE + value: $[variables['Build.SourceBranchName']] + - name: IS_TAG + value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] + - name: TARGET + value: $[variables['System.PullRequest.TargetBranch']] + steps: + # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python + # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable + - bash: | + WORKSPACE=$(python terraform/pipeline/workspace.py) + echo "##vso[task.setvariable variable=workspace;isOutput=true]$WORKSPACE" + + TAG_TYPE=$(python terraform/pipeline/tag.py) + echo "##vso[task.setvariable variable=tag_type;isOutput=true]$TAG_TYPE" + name: setvars + displayName: Determine deployment environment + env: + REASON: $(Build.Reason) + # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformInstaller#readme + - task: TerraformInstaller@0 + displayName: Install Terraform + inputs: + terraformVersion: 1.3.1 + # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme + - task: TerraformTaskV3@3 + displayName: Terraform init + inputs: + provider: azurerm + command: init + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow + commandOptions: -input=false + # service connection + backendServiceArm: deployer + # needs to match main.tf + backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform + backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmContainerName: tfstate + backendAzureRmKey: terraform.tfstate + - task: TerraformTaskV3@3 + displayName: Select environment + inputs: + provider: azurerm + command: custom + customCommand: workspace + commandOptions: select $(setvars.workspace) + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer + - task: TerraformTaskV3@3 + displayName: Terraform plan + inputs: + provider: azurerm + command: plan + # wait for lock to be released, in case being used by another pipeline run + # https://discuss.hashicorp.com/t/terraform-plan-wait-for-lock-to-be-released/6870/2 + commandOptions: -input=false -lock-timeout=5m -out=$(Build.ArtifactStagingDirectory)/tfplan + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer + # need to publish the tfplan to used by next stage if it's going to run + - publish: "$(Build.ArtifactStagingDirectory)" + displayName: "Publish tfplan for use in TerraformApply" + artifact: savedPlan + condition: | + or( + in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), + eq(variables['setvars.tag_type'], 'test'), + eq(variables['setvars.tag_type'], 'prod') + ) + - stage: TerraformApply + dependsOn: TerraformPlan + variables: + - name: workspace + value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] + - name: tag_type + value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] + # only run on dev, test, or prod branches OR if it's a tag for test or prod + condition: | + or( + in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), + eq(variables['tag_type'], 'test'), + eq(variables['tag_type'], 'prod') + ) + jobs: + - deployment: Apply + condition: succeeded() + environment: Approval + variables: + - name: workspace + value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.workspace'] ] + - name: tag_type + value: $[ stageDependencies.TerraformPlan.Plan.outputs['setvars.tag_type'] ] + strategy: + runOnce: + deploy: + steps: + - checkout: self + - download: current + displayName: "Download plan file published from TerraformPlan" + artifact: savedPlan + - task: TerraformInstaller@0 + displayName: Install Terraform + inputs: + terraformVersion: 1.3.1 + # https://github.com/microsoft/azure-pipelines-terraform/tree/main/Tasks/TerraformTask/TerraformTaskV3#readme + - task: TerraformTaskV3@3 + displayName: Terraform init + inputs: + provider: azurerm + command: init + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow + commandOptions: -input=false + # service connection + backendServiceArm: deployer + # needs to match main.tf + backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform + backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmContainerName: tfstate + backendAzureRmKey: terraform.tfstate + - task: TerraformTaskV3@3 + displayName: Select environment + inputs: + provider: azurerm + command: custom + customCommand: workspace + commandOptions: select $(workspace) + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer + - task: TerraformTaskV3@3 + displayName: Terraform apply + inputs: + provider: azurerm + command: apply + # (ditto the lock comment above) + commandOptions: -input=false -lock-timeout=5m $(Pipeline.Workspace)/savedPlan/tfplan + workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" + # service connection + environmentServiceNameAzureRM: deployer From 26b95e5441eaae213fa8ad97c503692e0f34840c Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 21:53:41 +0000 Subject: [PATCH 02/14] refactor: extract out agency-specific name --- terraform/pipeline/azure-pipelines.yml | 2 ++ terraform/pipeline/deploy.yml | 18 ++++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index dc04ced3..c1ea4714 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -26,3 +26,5 @@ pool: stages: - template: deploy.yml + parameters: + agency_card: courtesy-cards diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index f4b4ece8..c4d9921a 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -1,3 +1,13 @@ +parameters: + - name: agency_card + type: string + +variables: + - name: RESOURCE_GROUP + value: ${{ parameters.agency_card }}-eligibility-terraform + - name: STORAGE_ACCOUNT + value: ${{ replace(parameters.agency_card, '-', '') }}terraform + stages: - stage: TerraformPlan jobs: @@ -41,8 +51,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmResourceGroupName: $(RESOURCE_GROUP) + backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 @@ -123,8 +133,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: courtesy-cards-eligibility-terraform - backendAzureRmStorageAccountName: courtesycardsterraform + backendAzureRmResourceGroupName: $(RESOURCE_GROUP) + backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 From 393953a81bd100e3d71a45b998f828f4803714b7 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 21:54:40 +0000 Subject: [PATCH 03/14] chore: format template code --- terraform/pipeline/deploy.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index c4d9921a..a7b72be2 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -81,11 +81,11 @@ stages: displayName: "Publish tfplan for use in TerraformApply" artifact: savedPlan condition: | - or( - in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), - eq(variables['setvars.tag_type'], 'test'), - eq(variables['setvars.tag_type'], 'prod') - ) + or( + in(variables['Build.SourceBranchName'], 'dev', 'test', 'prod'), + eq(variables['setvars.tag_type'], 'test'), + eq(variables['setvars.tag_type'], 'prod') + ) - stage: TerraformApply dependsOn: TerraformPlan variables: From 01b2936216e0c6c529b9e2b37db5ed8384fd0967 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 22:14:12 +0000 Subject: [PATCH 04/14] refactor: variables don't work in templates, try pipeline --- terraform/pipeline/azure-pipelines.yml | 6 +++++- terraform/pipeline/deploy.yml | 18 +++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index c1ea4714..a6df05fb 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -24,7 +24,11 @@ pr: pool: vmImage: ubuntu-latest +variables: + agency_card: courtesy-cards + stages: - template: deploy.yml parameters: - agency_card: courtesy-cards + resource_group: $(agency_card)-eligibility-terraform + storage_account: ${{ replace(agency_card, '-', '') }}terraform diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index a7b72be2..bf89e83f 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -1,12 +1,8 @@ parameters: - - name: agency_card + - name: resource_group + type: string + - name: storage_account type: string - -variables: - - name: RESOURCE_GROUP - value: ${{ parameters.agency_card }}-eligibility-terraform - - name: STORAGE_ACCOUNT - value: ${{ replace(parameters.agency_card, '-', '') }}terraform stages: - stage: TerraformPlan @@ -51,8 +47,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: $(RESOURCE_GROUP) - backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) + backendAzureRmResourceGroupName: ${{ parameters.resource_group }} + backendAzureRmStorageAccountName: ${{ parameters.storage_account }} backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 @@ -133,8 +129,8 @@ stages: # service connection backendServiceArm: deployer # needs to match main.tf - backendAzureRmResourceGroupName: $(RESOURCE_GROUP) - backendAzureRmStorageAccountName: $(STORAGE_ACCOUNT) + backendAzureRmResourceGroupName: ${{ parameters.resource_group }} + backendAzureRmStorageAccountName: ${{ parameters.storage_account }} backendAzureRmContainerName: tfstate backendAzureRmKey: terraform.tfstate - task: TerraformTaskV3@3 From 100697e9b42a31879e6669970356faa6eb22691f Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 22:17:41 +0000 Subject: [PATCH 05/14] chore: fix syntax for using variable in expression --- terraform/pipeline/azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index a6df05fb..7bb3a281 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -31,4 +31,4 @@ stages: - template: deploy.yml parameters: resource_group: $(agency_card)-eligibility-terraform - storage_account: ${{ replace(agency_card, '-', '') }}terraform + storage_account: ${{ replace(variables.agency_card, '-', '') }}terraform From bc2379a397d37702da963f4c93ff95a4aa51746a Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 22:25:22 +0000 Subject: [PATCH 06/14] chore: prevent this noise from appearing in terraform plan --- terraform/suppress.arm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/suppress.arm.json b/terraform/suppress.arm.json index 95fd5b7d..51bfb6a3 100644 --- a/terraform/suppress.arm.json +++ b/terraform/suppress.arm.json @@ -3,7 +3,7 @@ "contentVersion": "1.0.0.0", "parameters": { "metricAlertID": { - "type": "string" + "type": "String" } }, "resources": [ From 2ea9a89e44378ceadb765fce9569a87454f3c1c6 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 23:08:36 +0000 Subject: [PATCH 07/14] refactor: variablize Terraform state resources --- terraform/main.tf | 4 ++-- terraform/pipeline/azure-pipelines.yml | 4 ++-- terraform/pipeline/deploy.yml | 8 ++++++++ terraform/variables.tf | 10 ++++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 0403446e..2aea8447 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -10,8 +10,8 @@ terraform { backend "azurerm" { # needs to match pipeline/azure-pipelines.yml - resource_group_name = "courtesy-cards-eligibility-terraform" - storage_account_name = "courtesycardsterraform" + resource_group_name = var.TF_RESOURCE_GROUP + storage_account_name = var.TF_STORAGE_ACCOUNT container_name = "tfstate" key = "terraform.tfstate" } diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/pipeline/azure-pipelines.yml index 7bb3a281..1c9a904d 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/pipeline/azure-pipelines.yml @@ -25,10 +25,10 @@ pool: vmImage: ubuntu-latest variables: - agency_card: courtesy-cards + agency_card: mst-courtesy-cards stages: - template: deploy.yml parameters: resource_group: $(agency_card)-eligibility-terraform - storage_account: ${{ replace(variables.agency_card, '-', '') }}terraform + storage_account: ${{ replace(variables.agency_card, '-', '') }}tf diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index bf89e83f..3202b164 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -17,6 +17,10 @@ stages: value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] - name: TARGET value: $[variables['System.PullRequest.TargetBranch']] + - name: TF_VAR_TF_RESOURCE_GROUP + value: ${{ parameters.resource_group }} + - name: TF_VAR_TF_STORAGE_ACCOUNT + value: ${{ parameters.storage_account }} steps: # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable @@ -89,6 +93,10 @@ stages: value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] - name: tag_type value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] + - name: TF_VAR_TF_RESOURCE_GROUP + value: ${{ parameters.resource_group }} + - name: TF_VAR_TF_STORAGE_ACCOUNT + value: ${{ parameters.storage_account }} # only run on dev, test, or prod branches OR if it's a tag for test or prod condition: | or( diff --git a/terraform/variables.tf b/terraform/variables.tf index f253713c..c62d0495 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,3 +33,13 @@ variable "IP_ADDRESS_WHITELIST_PROD" { type = list(string) default = [] } + +variable "TF_RESOURCE_GROUP" { + description = "The name of the resource group used for Terraform backend" + type = string +} + +variable "TF_STORAGE_ACCOUNT" { + description = "The name of the storage account used for Terraform backend" + type = string +} From ee93b21fd21a1e277710e36d73348c6cd85a616e Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Tue, 3 Oct 2023 23:39:00 +0000 Subject: [PATCH 08/14] refactor: extract out subscription name from init script --- terraform/init.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/terraform/init.sh b/terraform/init.sh index ece98f21..66add8e7 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -4,14 +4,15 @@ set -e ENV=$1 +SUBSCRIPTION=$2 -if [ $# -ne 1 ]; then - echo "Usage: $0 " +if [ $# -ne 2 ]; then + echo "Usage: $0 " exit 1 fi echo "Setting the subscription for the Azure CLI..." -az account set --subscription="MST IT" +az account set --subscription="$SUBSCRIPTION" printf "Intializing Terraform...\n\n" terraform init From 3803ffa222bfa0be8585eeebdc40ee40c13fc402 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 16:08:30 +0000 Subject: [PATCH 09/14] fix: move agency-specific backend configuration to command options backend configuration cannot use named values such as variables --- terraform/main.tf | 3 --- terraform/pipeline/deploy.yml | 10 ++++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 2aea8447..0343b806 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -9,9 +9,6 @@ terraform { } backend "azurerm" { - # needs to match pipeline/azure-pipelines.yml - resource_group_name = var.TF_RESOURCE_GROUP - storage_account_name = var.TF_STORAGE_ACCOUNT container_name = "tfstate" key = "terraform.tfstate" } diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 3202b164..70fec352 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -47,7 +47,10 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false + commandOptions: | + -input=false \ + -backend-config="resource_group=${{ parameters.resource_group }}" \ + -backend-config="storage_account=${{ parameters.storage_account }}" # service connection backendServiceArm: deployer # needs to match main.tf @@ -133,7 +136,10 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: -input=false + commandOptions: | + -input=false \ + -backend-config="resource_group=${{ parameters.resource_group }}" \ + -backend-config="storage_account=${{ parameters.storage_account }}" # service connection backendServiceArm: deployer # needs to match main.tf From 8af3d3252eab1b35e78375e94fc44a7760089157 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:13:44 +0000 Subject: [PATCH 10/14] feat(terraform): create agency-specific backend config for local dev the pipeline doesn't need the backend config block in main.tf --- terraform/init.sh | 10 ++++++---- terraform/mst/config.azurerm.tfbackend | 2 ++ terraform/mst/env | 1 + terraform/pipeline/deploy.yml | 10 ++-------- 4 files changed, 11 insertions(+), 12 deletions(-) create mode 100644 terraform/mst/config.azurerm.tfbackend create mode 100644 terraform/mst/env diff --git a/terraform/init.sh b/terraform/init.sh index 66add8e7..5f54cc1c 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -3,19 +3,21 @@ set -e -ENV=$1 -SUBSCRIPTION=$2 +ENV="$1" +AGENCY="$2" if [ $# -ne 2 ]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi +source "$AGENCY/env" + echo "Setting the subscription for the Azure CLI..." az account set --subscription="$SUBSCRIPTION" printf "Intializing Terraform...\n\n" -terraform init +terraform init -backend-config="$AGENCY/config.azurerm.tfbackend" printf "\n\nSelecting the Terraform workspace...\n" # matching logic in pipeline/workspace.py diff --git a/terraform/mst/config.azurerm.tfbackend b/terraform/mst/config.azurerm.tfbackend new file mode 100644 index 00000000..39ede935 --- /dev/null +++ b/terraform/mst/config.azurerm.tfbackend @@ -0,0 +1,2 @@ +resource_group_name="mst-courtesy-cards-eligibility-terraform" +storage_account_name="mstcourtesycardstf" diff --git a/terraform/mst/env b/terraform/mst/env new file mode 100644 index 00000000..29896c6d --- /dev/null +++ b/terraform/mst/env @@ -0,0 +1 @@ +SUBSCRIPTION="MST IT" diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 70fec352..3202b164 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -47,10 +47,7 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: | - -input=false \ - -backend-config="resource_group=${{ parameters.resource_group }}" \ - -backend-config="storage_account=${{ parameters.storage_account }}" + commandOptions: -input=false # service connection backendServiceArm: deployer # needs to match main.tf @@ -136,10 +133,7 @@ stages: command: init workingDirectory: "$(System.DefaultWorkingDirectory)/terraform" # https://developer.hashicorp.com/terraform/tutorials/automation/automate-terraform#automated-terraform-cli-workflow - commandOptions: | - -input=false \ - -backend-config="resource_group=${{ parameters.resource_group }}" \ - -backend-config="storage_account=${{ parameters.storage_account }}" + commandOptions: -input=false # service connection backendServiceArm: deployer # needs to match main.tf From e5545dd6c60d3b35f164d4a82b445ed64f863d8d Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:15:27 +0000 Subject: [PATCH 11/14] refactor(terraform): create workspace if it doesn't exist --- terraform/init.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/terraform/init.sh b/terraform/init.sh index 5f54cc1c..712a7797 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -20,11 +20,15 @@ printf "Intializing Terraform...\n\n" terraform init -backend-config="$AGENCY/config.azurerm.tfbackend" printf "\n\nSelecting the Terraform workspace...\n" + # matching logic in pipeline/workspace.py -if [ "$ENV" = "prod" ]; then - terraform workspace select default -else - terraform workspace select "$ENV" +WORKSPACE=$([[ "$ENV" == "prod" ]] && echo "default" || echo "$ENV") + +# if the workspace exists, this check will select it +WORKSPACE_EXISTS=$(terraform workspace select "$WORKSPACE" 2> /dev/null; echo $?) +# creating a new workspace also selects it +if [ "$WORKSPACE_EXISTS" -ne 0 ]; then + terraform workspace new "$WORKSPACE" fi echo "Done!" From 3bbc6a7d6db703455eee49d63932f3df3bd187ab Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:34:19 +0000 Subject: [PATCH 12/14] refactor(terraform): remove unneeded variables --- terraform/pipeline/deploy.yml | 8 -------- terraform/variables.tf | 10 ---------- 2 files changed, 18 deletions(-) diff --git a/terraform/pipeline/deploy.yml b/terraform/pipeline/deploy.yml index 3202b164..bf89e83f 100644 --- a/terraform/pipeline/deploy.yml +++ b/terraform/pipeline/deploy.yml @@ -17,10 +17,6 @@ stages: value: $[startsWith(variables['Build.SourceBranch'], 'refs/tags/')] - name: TARGET value: $[variables['System.PullRequest.TargetBranch']] - - name: TF_VAR_TF_RESOURCE_GROUP - value: ${{ parameters.resource_group }} - - name: TF_VAR_TF_STORAGE_ACCOUNT - value: ${{ parameters.storage_account }} steps: # set the workspace variable at runtime (rather than build time) so that all the necessary variables are available, and we can use Python # https://learn.microsoft.com/en-us/azure/devops/pipelines/process/set-variables-scripts?view=azure-devops&tabs=bash#about-tasksetvariable @@ -93,10 +89,6 @@ stages: value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.workspace'] ] - name: tag_type value: $[ dependencies.TerraformPlan.outputs['Plan.setvars.tag_type'] ] - - name: TF_VAR_TF_RESOURCE_GROUP - value: ${{ parameters.resource_group }} - - name: TF_VAR_TF_STORAGE_ACCOUNT - value: ${{ parameters.storage_account }} # only run on dev, test, or prod branches OR if it's a tag for test or prod condition: | or( diff --git a/terraform/variables.tf b/terraform/variables.tf index c62d0495..f253713c 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -33,13 +33,3 @@ variable "IP_ADDRESS_WHITELIST_PROD" { type = list(string) default = [] } - -variable "TF_RESOURCE_GROUP" { - description = "The name of the resource group used for Terraform backend" - type = string -} - -variable "TF_STORAGE_ACCOUNT" { - description = "The name of the storage account used for Terraform backend" - type = string -} From 3ad714ee9782123b8b63649b47d9e3b7fbe0d5a8 Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:41:36 +0000 Subject: [PATCH 13/14] refactor(terraform): rename local backend config file --- terraform/init.sh | 2 +- terraform/mst/{config.azurerm.tfbackend => local.tfbackend} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename terraform/mst/{config.azurerm.tfbackend => local.tfbackend} (100%) diff --git a/terraform/init.sh b/terraform/init.sh index 712a7797..bdb4bff2 100755 --- a/terraform/init.sh +++ b/terraform/init.sh @@ -17,7 +17,7 @@ echo "Setting the subscription for the Azure CLI..." az account set --subscription="$SUBSCRIPTION" printf "Intializing Terraform...\n\n" -terraform init -backend-config="$AGENCY/config.azurerm.tfbackend" +terraform init -backend-config="$AGENCY/local.tfbackend" printf "\n\nSelecting the Terraform workspace...\n" diff --git a/terraform/mst/config.azurerm.tfbackend b/terraform/mst/local.tfbackend similarity index 100% rename from terraform/mst/config.azurerm.tfbackend rename to terraform/mst/local.tfbackend From f94534bc9f34b1f8d806158aa2575d3b11ae6ded Mon Sep 17 00:00:00 2001 From: Angela Tran Date: Thu, 5 Oct 2023 17:45:53 +0000 Subject: [PATCH 14/14] refactor: move agency-specific pipeline into folder, simplify params --- terraform/{pipeline => mst}/azure-pipelines.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) rename terraform/{pipeline => mst}/azure-pipelines.yml (64%) diff --git a/terraform/pipeline/azure-pipelines.yml b/terraform/mst/azure-pipelines.yml similarity index 64% rename from terraform/pipeline/azure-pipelines.yml rename to terraform/mst/azure-pipelines.yml index 1c9a904d..90c1b67f 100644 --- a/terraform/pipeline/azure-pipelines.yml +++ b/terraform/mst/azure-pipelines.yml @@ -24,11 +24,8 @@ pr: pool: vmImage: ubuntu-latest -variables: - agency_card: mst-courtesy-cards - stages: - - template: deploy.yml + - template: ../pipeline/deploy.yml parameters: - resource_group: $(agency_card)-eligibility-terraform - storage_account: ${{ replace(variables.agency_card, '-', '') }}tf + resource_group: mst-courtesy-cards-eligibility-terraform + storage_account: mstcourtesycardstf