chore: add nightly gke cluster cleanup job (#9031)

.circleci/real_config.yml

@@ -2700,6 +2700,62 @@ jobs:
           mentions: <<parameters.slack-mentions>>
           channel: <<parameters.slack-channel>>
 
+  run-shared-cluster-cleanup:
+      parameters:
+        cluster-id:
+          type: string
+          default: ${GKE_CLUSTER_NAME}
+        region:
+          type: string
+          default: ${GKE_REGION}
+        gcloud-service-key:
+          default: GCLOUD_SERVICE_KEY
+          description: The gcloud service key
+          type: env_var_name
+        google-compute-zone:
+          default: GOOGLE_COMPUTE_ZONE
+          description: The Google compute zone to connect with via the gcloud CLI
+          type: env_var_name
+        google-project-id:
+          default: GOOGLE_PROJECT_ID
+          description: The Google project ID to connect with via the gcloud CLI
+          type: env_var_name
+      circleci_ip_ranges: true
+      docker:
+        - image: <<pipeline.parameters.docker-image>>
+      steps:
+        - set-cluster-id:
+              cluster-id: <<parameters.cluster-id>>
+        - gcloud/install:
+              version: "412.0.0"
+        - kubernetes/install-kubectl
+        - gcloud/initialize:
+            gcloud-service-key: <<parameters.gcloud-service-key>>
+            google-compute-zone: <<parameters.google-compute-zone>>
+            google-project-id: <<parameters.google-project-id>>
+        - run:
+            command: |
+              tries=5
+              until gcloud components install gke-gcloud-auth-plugin --quiet; do
+                if [[ $((--tries)) -eq 0 ]]; then
+                  exit 1
+                fi
+                sleep 15
+              done
+              echo "export USE_GKE_GCLOUD_AUTH_PLUGIN=True" >> $BASH_ENV
+            name: Install GKE auth plugin
+        - run:
+            name: Get Kubeconfig
+            command: gcloud container clusters get-credentials ${CLUSTER_ID} --project ${GOOGLE_PROJECT_ID} --region <<parameters.region>>
+        - run:
+            name: Delete GKE CI Cluster Namespaces
+            command: |
+              kubectl get namespace | grep -Eo "^test-cpu-[a-z0-9]+-[a-z0-9]+-[0-9]" | xargs -L1 kubectl delete namespace || true
+        - run:
+            name: Delete GCS CI Buckets
+            command: |
+              gsutil ls -p ${GOOGLE_PROJECT_ID} | grep -Eo "^gs://test-cpu-[a-z0-9]+-[a-z0-9]+-[0-9]-bucket" | xargs -L1 gsutil -m rm -r || true
+
 workflows:
   lint:
     jobs:
@@ -3583,6 +3639,10 @@ workflows:
           deployment-type: efs
           slack-mentions: "${SLACK_USER_ID}"
 
+      - run-shared-cluster-cleanup:
+          name: gke-cleanup
+          context: gcp-shared-cluster
+
   release:
     jobs:
       - build-helm: