Merge pull request #2441 from DFE-Digital/2041-configure-bigquery-in-…

…the-terraform-module

DFE analytics terraform module

.github/actions/deploy-environment/action.yml

@@ -37,6 +37,11 @@ runs:
       with:
         azure-credentials: ${{ inputs.azure-credentials }}
 
+    - uses: google-github-actions/auth@v2
+      with:
+        project_id: apply-for-qts-in-england
+        workload_identity_provider: projects/385922361840/locations/global/workloadIdentityPools/apply-for-qualified-teacher-sta3/providers/apply-for-qualified-teacher-sta3
+
     - name: Apply Terraform
       id: apply-terraform
       shell: bash

.github/workflows/deploy.yml

@@ -177,6 +177,10 @@ jobs:
     runs-on: ubuntu-latest
     if: contains(github.event.pull_request.labels.*.name, 'deploy')
     environment: review
+    permissions:
+      id-token: write
+      pull-requests: write
+
     steps:
       - uses: actions/checkout@v4
 
@@ -210,6 +214,9 @@ jobs:
       url: ${{ steps.deploy.outputs.url }}
     outputs:
       environment_name: ${{ matrix.environment }}
+    permissions:
+      id-token: write
+      pull-requests: write
 
     steps:
       - uses: actions/checkout@v4
@@ -230,6 +237,9 @@ jobs:
       name: production
       url: ${{ steps.deploy.outputs.url }}
     concurrency: deploy_production
+    permissions:
+      id-token: write
+      pull-requests: write
 
     steps:
       - uses: actions/checkout@v4

config/initializers/dfe_analytics.rb

@@ -13,4 +13,6 @@
       disabled_by_default = Rails.env.development?
       ENV.fetch("BIGQUERY_DISABLE", disabled_by_default.to_s) != "true"
     end
+
+  config.azure_federated_auth = ENV.include? "GOOGLE_CLOUD_CREDENTIALS"
 end

docs/setting-up-analytics.md

@@ -27,6 +27,8 @@ in the free tier
 
 ## 3. Create a data set and table
 
+Update: The [dfe_analytics](https://github.com/DFE-Digital/terraform-modules/tree/main/aks/dfe_analytics) terraform module now automates these steps.
+
 You should create separate data sets for each environment (dev/preprod/prod).
 
 1. Select the BigQuery instance
@@ -189,6 +191,8 @@ If you edit as text, you can paste this:
 
 ## 4. Create custom roles
 
+Update: The [dfe_analytics](https://github.com/DFE-Digital/terraform-modules/tree/main/aks/dfe_analytics) terraform module now automates these steps.
+
 1. Go to IAM and Admin settings > Roles
 1. Click on "+ Create role"
 1. Create the 3 roles outlined below
@@ -321,13 +325,17 @@ If you edit as text, you can paste this:
 
 ## 5. Create an appender service account
 
+Update: The [dfe_analytics](https://github.com/DFE-Digital/terraform-modules/tree/main/aks/dfe_analytics) terraform module now automates these steps.
+
 1. Go to [IAM and Admin settings > Create service account](https://console.cloud.google.com/projectselector/iam-admin/serviceaccounts/create?supportedpurview=project)
 1. Name it like "Appender NAME_OF_SERVICE ENVIRONMENT", so "Appender Apply Local"
 1. Add a description, like "Used when developing locally."
 1. Grant the service account access to the project, use the "BigQuery Appender Custom" role you set up earlier
 
 ## 6. Get an API JSON key :key:
 
+Update: The [dfe_analytics](https://github.com/DFE-Digital/terraform-modules/tree/main/aks/dfe_analytics) terraform module now automates these steps.
+
 1. Access the service account you previously setup
 1. Go to the keys tab, click on "Add key > Create new key"
 1. Create a JSON private key
@@ -336,6 +344,8 @@ The full contents of this JSON file is your `BIGQUERY_API_JSON_KEY`.
 
 ## 6. Set up environment variables
 
+Update: The [dfe_analytics](https://github.com/DFE-Digital/terraform-modules/tree/main/aks/dfe_analytics) terraform module now automates these steps.
+
 Putting the previous things together, to finish setting up `dfe-analytics`, you
 need these environment variables:
 

terraform/application/application.tf

@@ -19,9 +19,9 @@ module "application_configuration" {
     {
       HOSTING_ENVIRONMENT = local.environment
 
-      BIGQUERY_PROJECT_ID = "apply-for-qts-in-england"
-      BIGQUERY_DATASET    = "events_${var.app_environment}"
-      BIGQUERY_TABLE_NAME = "events"
+      BIGQUERY_PROJECT_ID = var.bigquery_federated_auth ? module.dfe_analytics[0].bigquery_project_id : "apply-for-qts-in-england"
+      BIGQUERY_DATASET    = var.bigquery_federated_auth ? module.dfe_analytics[0].bigquery_dataset : "events_${var.app_environment}"
+      BIGQUERY_TABLE_NAME = var.bigquery_federated_auth ? module.dfe_analytics[0].bigquery_table_name : "events"
   })
 
   secret_key_vault_short = "app"
@@ -32,6 +32,8 @@ module "application_configuration" {
     AZURE_STORAGE_ACCOUNT_NAME = azurerm_storage_account.uploads.name
     AZURE_STORAGE_ACCESS_KEY   = azurerm_storage_account.uploads.primary_access_key
     AZURE_STORAGE_CONTAINER    = azurerm_storage_container.uploads.name
+
+    GOOGLE_CLOUD_CREDENTIALS = var.bigquery_federated_auth ? module.dfe_analytics[0].google_cloud_credentials : null
   }
 }
 
@@ -74,5 +76,6 @@ module "worker_application" {
   command       = ["bundle", "exec", "sidekiq", "-C", "./config/sidekiq.yml"]
   probe_command = ["pgrep", "-f", "sidekiq"]
 
-  enable_logit = var.enable_logit
+  enable_logit   = var.enable_logit
+  enable_gcp_wif = var.bigquery_federated_auth ? true : null
 }

terraform/application/config/review/variables.tfvars.json

@@ -5,5 +5,6 @@
   "enable_logit": true,
   "enable_monitoring": false,
   "namespace": "tra-development",
-  "uploads_storage_environment_tag": "Test"
+  "uploads_storage_environment_tag": "Test",
+  "bigquery_federated_auth": true
 }

terraform/application/dfe_analytics.tf

@@ -0,0 +1,15 @@
+provider "google" {
+  project = "apply-for-qts-in-england"
+}
+
+module "dfe_analytics" {
+  count  = var.bigquery_federated_auth ? 1 : 0
+  source = "./vendor/modules/dfe-terraform-modules//aks/dfe_analytics"
+
+  azure_resource_prefix = var.azure_resource_prefix
+  cluster               = var.cluster
+  namespace             = var.namespace
+  service_short         = var.service_short
+  environment           = "${var.app_environment}${var.app_suffix}"
+  gcp_dataset           = "events_${var.app_environment}"
+}

terraform/application/variables.tf

@@ -85,6 +85,12 @@ variable "uploads_storage_account_name" {
   default = null
 }
 
+variable "bigquery_federated_auth" {
+  type        = bool
+  default     = false
+  description = "Configure environment variable to let dfe-analytics use federated authentication"
+}
+
 locals {
   environment_variables = yamldecode(file("${path.module}/config/${var.app_environment}/variables.yml"))
 }