infra/gcp: add k8s-project-metrics special-case

k8s-metrics is a low-traffic GCS bucket for the project, setup permissions to trial moving it from one org to the other. The steps for this process are going to look something like: - allow google.com prow to write to the new bucket - allow k8s-infra-prow-build-trusted to write to the new bucket - allow humans to own the new bucket - sync contents from old bucket to new bucket - setup canary job on k8s-infra-prow-build-trusted that writes to the new bucket to confirm permissions are correct for executing bigquery queries that use the k8s-gubernator:builds dataset and writing to the new bucket - remove canary job / move old job to k8s-infra-prow-build-trusted - delete old bucket - rename new bucket to old bucket
kubernetes · Aug 2, 2021 · adbd4e2 · adbd4e2
1 parent e201101
commit adbd4e2
Show file tree

Hide file tree

Showing 3 changed files with 82 additions and 21 deletions.
diff --git a/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf b/infra/gcp/clusters/projects/k8s-infra-prow-build-trusted/prow-build-trusted/main.tf
@@ -24,15 +24,16 @@ This file defines:
 
 locals {
   project_id                   = "k8s-infra-prow-build-trusted"
-  cluster_name                 = "prow-build-trusted"  // The name of the cluster defined in this file
-  cluster_location             = "us-central1"         // The GCP location (region or zone) where the cluster should be created
-  bigquery_location            = "US"                  // The bigquery specific location where the dataset should be created
-  pod_namespace                = "test-pods"           // MUST match whatever prow is configured to use when it schedules to this cluster
+  cluster_name                 = "prow-build-trusted"   // The name of the cluster defined in this file
+  cluster_location             = "us-central1"          // The GCP location (region or zone) where the cluster should be created
+  bigquery_location            = "US"                   // The bigquery specific location where the dataset should be created
+  pod_namespace                = "test-pods"            // MUST match whatever prow is configured to use when it schedules to this cluster
 
   // Service Accounts in ${pod_namespace} (usable via Workload Identity)
-  cluster_sa_name              = "prow-build-trusted"           // Pods use this by default
-  gcb_builder_sa_name          = "gcb-builder"                  // Allowed to run GCB builds and push to GCS buckets
-  prow_deployer_sa_name        = "prow-deployer"                // Allowed to deploy to prow build clusters
+  cluster_sa_name              = "prow-build-trusted"   // Pods use this by default
+  gcb_builder_sa_name          = "gcb-builder"          // Allowed to run GCB builds and push to GCS buckets
+  prow_deployer_sa_name        = "prow-deployer"        // Allowed to deploy to prow build clusters
+  k8s_metrics_sa_name          = "k8s-metrics"          // Allowed to write to gs://k8s-metrics
 }
 
 data "google_organization" "org" {
@@ -117,6 +118,27 @@ resource "google_service_account_iam_policy" "prow_deployer_sa_iam" {
   service_account_id = google_service_account.prow_deployer_sa.name
   policy_data        = data.google_iam_policy.prow_deployer_sa_workload_identity.policy_data
 }
+// Create GCP SA for jobs that write to gs://k8s-metrics and gs://k8s-project-metrics
+resource "google_service_account" "k8s_metrics_sa" {
+  project      = local.project_id
+  account_id   = local.k8s_metrics_sa_name
+  display_name = local.k8s_metrics_sa_name
+}
+// Allow pods using the build cluster KSA to use the GCP SA via workload identity
+data "google_iam_policy" "k8s_metrics_sa_workload_identity" {
+  binding {
+    role = "roles/iam.workloadIdentityUser"
+
+    members = [
+      "serviceAccount:${local.project_id}.svc.id.goog[${local.pod_namespace}/${local.k8s_metrics_sa_name}]",
+    ]
+  }
+}
+// Authoritative iam-policy: replaces any existing policy attached to this service_account
+resource "google_service_account_iam_policy" "k8s_metrics_sa_iam" {
+  service_account_id = google_service_account.k8s_metrics_sa.name
+  policy_data        = data.google_iam_policy.k8s_metrics_sa_workload_identity.policy_data
+}
 
 resource "google_project_iam_member" "prow_deployer_for_prow_build_trusted" {
   project = local.project_id

diff --git a/...-prow-build-trusted/prow-build-trusted/resources/test-pods/test-pods-serviceaccounts.yaml b/...-prow-build-trusted/prow-build-trusted/resources/test-pods/test-pods-serviceaccounts.yaml
@@ -1,3 +1,4 @@
+# Prow service accounts
 ---
 apiVersion: v1
 kind: ServiceAccount
@@ -7,20 +8,30 @@ metadata:
   name: prow-build-trusted
   namespace: test-pods
 ---
+kind: ServiceAccount
 apiVersion: v1
+metadata:
+  annotations:
+    iam.gke.io/gcp-service-account: prow-deployer@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
+  name: prow-deployer
+  namespace: test-pods
+---
 kind: ServiceAccount
+apiVersion: v1
 metadata:
   annotations:
-    iam.gke.io/gcp-service-account: k8s-infra-gcp-auditor@kubernetes-public.iam.gserviceaccount.com
-  name: k8s-infra-gcp-auditor
+    iam.gke.io/gcp-service-account: k8s-metrics@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
+  name: k8s-metrics
   namespace: test-pods
+
+# Infrastructure management service accounts
 ---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   annotations:
-    iam.gke.io/gcp-service-account: gcb-builder@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
-  name: gcb-builder
+    iam.gke.io/gcp-service-account: k8s-infra-gcp-auditor@kubernetes-public.iam.gserviceaccount.com
+  name: k8s-infra-gcp-auditor
   namespace: test-pods
 ---
 kind: ServiceAccount
@@ -38,30 +49,33 @@ metadata:
     iam.gke.io/gcp-service-account: [email protected]
   name: gsuite-groups-manager
   namespace: test-pods
+
+# Image promotion service accounts
 ---
 kind: ServiceAccount
 apiVersion: v1
 metadata:
   annotations:
-    iam.gke.io/gcp-service-account: prow-deployer@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
-  name: prow-deployer
+    iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod.iam.gserviceaccount.com
+  name: k8s-infra-gcr-promoter
   namespace: test-pods
 ---
-# Image promotion service accounts
 kind: ServiceAccount
 apiVersion: v1
 metadata:
   annotations:
-    iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod.iam.gserviceaccount.com
-  name: k8s-infra-gcr-promoter
+    iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod-bak.iam.gserviceaccount.com
+  name: k8s-infra-gcr-promoter-bak
   namespace: test-pods
+
+# Staging service accounts
 ---
-kind: ServiceAccount
 apiVersion: v1
+kind: ServiceAccount
 metadata:
   annotations:
-    iam.gke.io/gcp-service-account: k8s-infra-gcr-promoter@k8s-artifacts-prod-bak.iam.gserviceaccount.com
-  name: k8s-infra-gcr-promoter-bak
+    iam.gke.io/gcp-service-account: gcb-builder@k8s-infra-prow-build-trusted.iam.gserviceaccount.com
+  name: gcb-builder
   namespace: test-pods
 ---
 kind: ServiceAccount

diff --git a/infra/gcp/ensure-main-project.sh b/infra/gcp/ensure-main-project.sh
@@ -396,11 +396,36 @@ function ensure_prow_special_cases {
         return 1
     fi
     local project="${1}"
+
+    local bucket principal secret
+
     color 6 "Special case: ensuring k8s-infra-ci-robot-github-token accessible by k8s-infra-prow-build-trusted"
-    local principal secret
     principal="serviceAccount:$(svc_acct_email "k8s-infra-prow-build-trusted" "kubernetes-external-secrets")"
     secret=$(secret_full_name "${project}" "k8s-infra-ci-robot-github-token")
-    ensure_secret_role_binding "${secret}" "${principal}" "roles/secretmanager.secretAccessor"
+    ensure_secret_role_binding "${secret}" "${principal}" "roles/secretmanager.secretAccessor" 2>&1 | indent
+
+    color 6 "Special case: ensuring gs://k8s-metrics-canary exists for gs://k8s-metrics migration"
+    (
+      bucket="gs://k8s-project-metrics"
+      owners="[email protected]"
+      local old_service_account="[email protected]"
+
+      ensure_public_gcs_bucket "${project}" "${bucket}"
+      ensure_gcs_bucket_auto_deletion "${bucket}" "365" # match gs://k8s-metrics
+      # GCS admins can admin all GCS buckets
+      empower_gcs_admins "${project}" "${bucket}"
+      # bucket owners can admin this bucket
+      empower_group_to_admin_gcs_bucket "${owners}" "${bucket}"
+      # TODO(spiffxp): copy pasted to flip to ensure_removed when migrated
+      # k8s-prow-builds can write to this bucket
+      principal="serviceAccount:${old_service_account}"
+      ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin"
+      ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter"
+      # k8s-infra-prow-build-trusted can write to this bucket
+      principal="serviceAccount:$(svc_acc_email "k8s-infra-prow-build-trusted" "k8s-metrics")"
+      ensure_gcs_role_binding "${bucket}" "${principal}" "objectAdmin"
+      ensure_gcs_role_binding "${bucket}" "${principal}" "legacyBucketWriter"
+    ) 2>&1 | indent
 }
 
 function ensure_main_project() {