From 6cfbb4929e88ce93356f7106363c74c51a5e8890 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 11 Jul 2024 14:39:25 +0200 Subject: [PATCH 1/4] terraform, gcp: use dynamic budget alerts based on last month --- docs/howto/budget-alerts.md | 6 ------ terraform/gcp/budget-alerts.tf | 19 ++++++++----------- terraform/gcp/projects/2i2c-uk.tfvars | 4 +--- terraform/gcp/projects/awi-ciroh-2.tfvars | 3 +-- terraform/gcp/projects/awi-ciroh.tfvars | 1 - .../gcp/projects/catalystproject-latam.tfvars | 4 +--- terraform/gcp/projects/cloudbank.tfvars | 3 +-- .../gcp/projects/cluster.tfvars.template | 1 - terraform/gcp/projects/hhmi.tfvars | 4 +--- terraform/gcp/projects/leap.tfvars | 1 - terraform/gcp/projects/linked-earth.tfvars | 1 - terraform/gcp/projects/pangeo-hubs.tfvars | 1 - terraform/gcp/projects/pilot-hubs.tfvars | 4 +--- terraform/gcp/variables.tf | 16 ---------------- 14 files changed, 14 insertions(+), 54 deletions(-) diff --git a/docs/howto/budget-alerts.md b/docs/howto/budget-alerts.md index b6a3850131..3f6ded7439 100644 --- a/docs/howto/budget-alerts.md +++ b/docs/howto/budget-alerts.md @@ -29,11 +29,5 @@ Then edit the following variables in the relevant `.tfvars` file for the cluster This will open a pane that gives you the Billing Account ID. - For accounts that we don't manage, the process is the same but _we may not have permission to view the Billing Account ID_. In this case, we cannot enable budget alerting for this project. -- **Set `budget_alert_amount`.** - Current practice is to set this to the average expenditure of the last 3 months, plus 20%. - You can find values to calculate that in the [Billing Reports console](https://console.cloud.google.com/billing/0157F7-E3EA8C-25AC3C/reports?organizationId=184174754493&project=two-eye-two-see). - _Make sure you select only the project you are interested in from the Projects field in the Filters pane on the right side of the screen._ - - If you are setting this up for a new cluster, you obviously don't have this information yet! - Instead, set the value to `400` and we can adjust as the community begins to use it. With these variables set, you are ready to open a PR and perform a terraform apply! diff --git a/terraform/gcp/budget-alerts.tf b/terraform/gcp/budget-alerts.tf index b6b486e788..afa51ac990 100644 --- a/terraform/gcp/budget-alerts.tf +++ b/terraform/gcp/budget-alerts.tf @@ -28,13 +28,11 @@ resource "google_billing_budget" "budget" { # This is a bug in the google provider / budgets API https://github.com/hashicorp/terraform-provider-google/issues/8444 projects = ["projects/${data.google_project.project.number}"] credit_types_treatment = "INCLUDE_ALL_CREDITS" + calendar_period = "MONTH" } amount { - specified_amount { - currency_code = var.budget_alert_currency - units = var.budget_alert_amount - } + last_period_amount = true } all_updates_rule { @@ -43,19 +41,18 @@ resource "google_billing_budget" "budget" { ] disable_default_iam_recipients = true } + # NOTE: These threshold_rules *MUST BE ORDERED BY threshold_percent* in ascending order! # If not, we'll run into https://github.com/hashicorp/terraform-provider-google/issues/8444 # and terraform apply won't be clean. threshold_rules { - # Alert when *actual* spend reached 80% of budget - threshold_percent = 1.0 + # Alert when *actual* spend reached 120% of budget (last month's spend) + threshold_percent = 1.2 spend_basis = "CURRENT_SPEND" } threshold_rules { - # Alert when *forecasted* spend is about to blow over our budget - # Adding the extra 1% to help terraform not redo this each time. - threshold_percent = 1.01 + # Alert when *forecasted* spend reached 120% of budget (last month's spend) + threshold_percent = 1.2 spend_basis = "FORECASTED_SPEND" } - -} \ No newline at end of file +} diff --git a/terraform/gcp/projects/2i2c-uk.tfvars b/terraform/gcp/projects/2i2c-uk.tfvars index 144375d37a..522037e688 100644 --- a/terraform/gcp/projects/2i2c-uk.tfvars +++ b/terraform/gcp/projects/2i2c-uk.tfvars @@ -4,9 +4,7 @@ project_id = "two-eye-two-see-uk" zone = "europe-west2-b" region = "europe-west2" -# This is the average of total costs for Apr -> Jun 2024 +20% in USD -budget_alert_amount = "830" -billing_account_id = "0157F7-E3EA8C-25AC3C" +billing_account_id = "0157F7-E3EA8C-25AC3C" k8s_versions = { min_master_version : "1.29.1-gke.1589018", diff --git a/terraform/gcp/projects/awi-ciroh-2.tfvars b/terraform/gcp/projects/awi-ciroh-2.tfvars index 6a9717a33e..a92bc7ae91 100644 --- a/terraform/gcp/projects/awi-ciroh-2.tfvars +++ b/terraform/gcp/projects/awi-ciroh-2.tfvars @@ -10,9 +10,8 @@ filestores = { "filestore" = { capacity_gb = 5939 } } -# This project does not have cloud costs passed through by 2i2c +# Cloud costs for this project are not passed through by 2i2c budget_alert_enabled = false -budget_alert_amount = "" billing_account_id = "" k8s_versions = { diff --git a/terraform/gcp/projects/awi-ciroh.tfvars b/terraform/gcp/projects/awi-ciroh.tfvars index 8213c9d3e1..654ca63e30 100644 --- a/terraform/gcp/projects/awi-ciroh.tfvars +++ b/terraform/gcp/projects/awi-ciroh.tfvars @@ -1,6 +1,5 @@ budget_alert_enabled = false billing_account_id = "0157F7-E3EA8C-25AC3C" -budget_alert_amount = "800" prefix = "awi-ciroh" project_id = "awi-ciroh" diff --git a/terraform/gcp/projects/catalystproject-latam.tfvars b/terraform/gcp/projects/catalystproject-latam.tfvars index 32718703cf..2255a1bd67 100644 --- a/terraform/gcp/projects/catalystproject-latam.tfvars +++ b/terraform/gcp/projects/catalystproject-latam.tfvars @@ -4,9 +4,7 @@ region = "southamerica-east1" zone = "southamerica-east1-c" enable_network_policy = true -# This is the average of total costs for Apr -> Jun 2024 +20% in USD -budget_alert_amount = "1672" -billing_account_id = "0157F7-E3EA8C-25AC3C" +billing_account_id = "0157F7-E3EA8C-25AC3C" k8s_versions = { min_master_version : "1.29.1-gke.1589018", diff --git a/terraform/gcp/projects/cloudbank.tfvars b/terraform/gcp/projects/cloudbank.tfvars index 7400de08e7..7dfc8a5780 100644 --- a/terraform/gcp/projects/cloudbank.tfvars +++ b/terraform/gcp/projects/cloudbank.tfvars @@ -5,10 +5,9 @@ zone = "us-central1-b" region = "us-central1" regional_cluster = false -# We don't have enough access to enable this +# Cloud costs for this project are not passed through by 2i2c budget_alert_enabled = false billing_account_id = "" -budget_alert_amount = "" k8s_versions = { # NOTE: This isn't a regional cluster / highly available cluster, when diff --git a/terraform/gcp/projects/cluster.tfvars.template b/terraform/gcp/projects/cluster.tfvars.template index 380735969e..7e43fe91b5 100644 --- a/terraform/gcp/projects/cluster.tfvars.template +++ b/terraform/gcp/projects/cluster.tfvars.template @@ -13,7 +13,6 @@ region = "{{ cluster_region }}" # Config required to enable automatic budget alerts to be sent to support@2i2c.org budget_alert_enabled = false -budget_alert_amount = "" billing_account_id = "" # TODO: Before applying this, identify a k8s version to specify. Pick the latest diff --git a/terraform/gcp/projects/hhmi.tfvars b/terraform/gcp/projects/hhmi.tfvars index bc346bb06b..5bef250ded 100644 --- a/terraform/gcp/projects/hhmi.tfvars +++ b/terraform/gcp/projects/hhmi.tfvars @@ -6,9 +6,7 @@ region = "us-west2" core_node_machine_type = "n2-highmem-4" -# This is the average of total costs for Apr -> Jun 2024 +20% in USD -budget_alert_amount = "797" -billing_account_id = "0157F7-E3EA8C-25AC3C" +billing_account_id = "0157F7-E3EA8C-25AC3C" k8s_versions = { min_master_version : "1.29.1-gke.1589020", diff --git a/terraform/gcp/projects/leap.tfvars b/terraform/gcp/projects/leap.tfvars index 42f141ff3e..402eec64d3 100644 --- a/terraform/gcp/projects/leap.tfvars +++ b/terraform/gcp/projects/leap.tfvars @@ -6,7 +6,6 @@ core_node_machine_type = "n2-highmem-4" # Cloud costs for this project are not passed through by 2i2c budget_alert_enabled = false -budget_alert_amount = "" billing_account_id = "" k8s_versions = { diff --git a/terraform/gcp/projects/linked-earth.tfvars b/terraform/gcp/projects/linked-earth.tfvars index bb67cf7b23..d647638ff9 100644 --- a/terraform/gcp/projects/linked-earth.tfvars +++ b/terraform/gcp/projects/linked-earth.tfvars @@ -6,7 +6,6 @@ region = "us-central1" # Cloud costs for this project are not passed through by 2i2c budget_alert_enabled = false -budget_alert_amount = "" billing_account_id = "" k8s_versions = { diff --git a/terraform/gcp/projects/pangeo-hubs.tfvars b/terraform/gcp/projects/pangeo-hubs.tfvars index 80759282e3..fe1b852a52 100644 --- a/terraform/gcp/projects/pangeo-hubs.tfvars +++ b/terraform/gcp/projects/pangeo-hubs.tfvars @@ -29,7 +29,6 @@ enable_logging = false # We don't have enough rights to make billing alerts budget_alert_enabled = false -budget_alert_amount = "" billing_account_id = "" k8s_versions = { diff --git a/terraform/gcp/projects/pilot-hubs.tfvars b/terraform/gcp/projects/pilot-hubs.tfvars index 80e2b6e1ef..a87248ef66 100644 --- a/terraform/gcp/projects/pilot-hubs.tfvars +++ b/terraform/gcp/projects/pilot-hubs.tfvars @@ -5,9 +5,7 @@ zone = "us-central1-b" region = "us-central1" regional_cluster = false -# This is the average of total costs for Apr -> Jun 2024 +20% in USD -budget_alert_amount = "1880" -billing_account_id = "0157F7-E3EA8C-25AC3C" +billing_account_id = "0157F7-E3EA8C-25AC3C" k8s_versions = { # NOTE: This isn't a regional cluster / highly available cluster, when diff --git a/terraform/gcp/variables.tf b/terraform/gcp/variables.tf index 9f3ebb3225..03d35081f1 100644 --- a/terraform/gcp/variables.tf +++ b/terraform/gcp/variables.tf @@ -31,22 +31,6 @@ variable "billing_account_id" { EOT } -variable "budget_alert_currency" { - type = string - default = "USD" - description = <<-EOT - Currency used for budget alerts. - EOT -} - -variable "budget_alert_amount" { - type = string - description = <<-EOT - Amount of *forecasted spend* at which to send a billing alert. Current practice - is to set this to the average of the last 3 months expenditure + 20%. - EOT -} - variable "budget_alert_enabled" { type = bool default = true From cc6b57182e57b09d5d770a282abca8678298d1d2 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 11 Jul 2024 14:41:52 +0200 Subject: [PATCH 2/4] terraform, gcp: let budget alerts display a reference to related project --- terraform/gcp/budget-alerts.tf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/terraform/gcp/budget-alerts.tf b/terraform/gcp/budget-alerts.tf index afa51ac990..ad739f8ea1 100644 --- a/terraform/gcp/budget-alerts.tf +++ b/terraform/gcp/budget-alerts.tf @@ -1,13 +1,14 @@ -# Alerts sent to support@2i2c.org for things that *will go bad* in the future -# if left unattended. Should *not* be used for immediate outages +# Alerts sent to support+budget-${var.prefix}@2i2c.org for things that *will go +# bad* in the future if left unattended. Should *not* be used for immediate +# outages resource "google_monitoring_notification_channel" "support_email" { count = var.budget_alert_enabled ? 1 : 0 project = var.project_id - display_name = "support@2i2c.org email" + display_name = "Email support+budget-${var.prefix}@2i2c.org" type = "email" labels = { - email_address = "support@2i2c.org" + email_address = "support+budget-${var.prefix}@2i2c.org" } } @@ -16,11 +17,12 @@ data "google_project" "project" { } # Need to explicitly enable https://console.cloud.google.com/apis/library/billingbudgets.googleapis.com?project=two-eye-two-see +# resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/billing_budget resource "google_billing_budget" "budget" { count = var.budget_alert_enabled ? 1 : 0 billing_account = var.billing_account_id - display_name = "Billing alert" + display_name = "Auto-adjusting budget for ${var.prefix}" budget_filter { # Use project number here, as project_name seems to be converted internally to number From bc1f9e125e14bd91be6c144ba723a9c5ef5c5be2 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 11 Jul 2024 15:04:24 +0200 Subject: [PATCH 3/4] terraform, gcp: micro refactor by reordering in one file --- terraform/gcp/budget-alerts.tf | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/terraform/gcp/budget-alerts.tf b/terraform/gcp/budget-alerts.tf index ad739f8ea1..038fc71a23 100644 --- a/terraform/gcp/budget-alerts.tf +++ b/terraform/gcp/budget-alerts.tf @@ -2,6 +2,10 @@ # bad* in the future if left unattended. Should *not* be used for immediate # outages +data "google_project" "project" { + project_id = var.project_id +} + resource "google_monitoring_notification_channel" "support_email" { count = var.budget_alert_enabled ? 1 : 0 project = var.project_id @@ -12,10 +16,6 @@ resource "google_monitoring_notification_channel" "support_email" { } } -data "google_project" "project" { - project_id = var.project_id -} - # Need to explicitly enable https://console.cloud.google.com/apis/library/billingbudgets.googleapis.com?project=two-eye-two-see # resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/billing_budget resource "google_billing_budget" "budget" { @@ -24,6 +24,13 @@ resource "google_billing_budget" "budget" { billing_account = var.billing_account_id display_name = "Auto-adjusting budget for ${var.prefix}" + all_updates_rule { + monitoring_notification_channels = [ + google_monitoring_notification_channel.support_email[0].id, + ] + disable_default_iam_recipients = true + } + budget_filter { # Use project number here, as project_name seems to be converted internally to number # If we don't do this, `terraform apply` is not clean @@ -37,13 +44,6 @@ resource "google_billing_budget" "budget" { last_period_amount = true } - all_updates_rule { - monitoring_notification_channels = [ - google_monitoring_notification_channel.support_email[0].id, - ] - disable_default_iam_recipients = true - } - # NOTE: These threshold_rules *MUST BE ORDERED BY threshold_percent* in ascending order! # If not, we'll run into https://github.com/hashicorp/terraform-provider-google/issues/8444 # and terraform apply won't be clean. From f86823650c20706e01a3cc4754e27c4e782c7e1f Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 11 Jul 2024 15:13:37 +0200 Subject: [PATCH 4/4] terraform, gcp: rename budget resource as a workaround Apparently terraform fails to migrate a budget with a fixed number to a budget referencing the last month, but recreating the budget worked fine though. By renaming this resource we accomplished that. This is what was shown repeatedly after each terraform apply without forcing the re-creation: ``` Terraform will perform the following actions: # google_billing_budget.budget[0] will be updated in-place ~ resource "google_billing_budget" "budget" { id = "censored" name = "censored" # (3 unchanged attributes hidden) ~ amount { ~ last_period_amount = false -> true - specified_amount { - currency_code = "USD" -> null - nanos = 0 -> null # (1 unchanged attribute hidden) } } # (4 unchanged blocks hidden) } Plan: 0 to add, 1 to change, 0 to destroy. ```` --- terraform/gcp/budget-alerts.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/gcp/budget-alerts.tf b/terraform/gcp/budget-alerts.tf index 038fc71a23..b46a8cb3fa 100644 --- a/terraform/gcp/budget-alerts.tf +++ b/terraform/gcp/budget-alerts.tf @@ -18,7 +18,7 @@ resource "google_monitoring_notification_channel" "support_email" { # Need to explicitly enable https://console.cloud.google.com/apis/library/billingbudgets.googleapis.com?project=two-eye-two-see # resource ref: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/billing_budget -resource "google_billing_budget" "budget" { +resource "google_billing_budget" "budgets" { count = var.budget_alert_enabled ? 1 : 0 billing_account = var.billing_account_id