Merge branch 'GoogleCloudPlatform:main' into main

.ci/gcb-generate-diffs-new.yml

@@ -255,7 +255,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-tpg-vcr-test
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
       waitFor: ["diff"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -318,5 +318,3 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
-    - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest
-      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION

.ci/gcb-push-downstream.yml

@@ -206,7 +206,6 @@ steps:
         - "GOOGLE_SERVICE_ACCOUNT"
         - "SA_KEY"
         - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"
-        - "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"
       env:
         - "COMMIT_SHA=$COMMIT_SHA"
         - "GOOGLE_REGION=us-central1"
@@ -253,5 +252,3 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
-    - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest
-      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION

.ci/gcb-vcr-nightly.yml

@@ -3,7 +3,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-vcr-nightly
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh'
-      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
+      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
       args:
           - $BUILD_ID
 
@@ -41,5 +41,3 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
-    - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest
-      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION

.ci/infra/terraform/README.md

@@ -48,6 +48,7 @@ After applying this configuration:
 - Create a `support@` group in the Google Workspace Admin Console, add new service account as a member, and make it an owner
 - Enroll in Cloud Armor Managed Protection Plus tier
 - Add Cloud Identity Premium Plan to the Google Workspace domain
+- Perform the Privileged Access Manager set-up https://pantheon.corp.google.com/iam-admin/pam/setup
 
 Quotas that will need to be adjusted to support all tests:
 - Project quota for the new service account

.ci/infra/terraform/main.tf

@@ -33,6 +33,12 @@ resource "google_organization_iam_member" "sa_access_boundary_admin" {
   member = google_service_account.sa.member
 }
 
+resource "google_organization_iam_member" "sa_apphub_admin" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/apphub.admin"
+  member = google_service_account.sa.member
+}
+
 resource "google_organization_iam_member" "sa_assuredworkloads_admin" {
   org_id = data.google_organization.org.org_id
   role   = "roles/assuredworkloads.admin"
@@ -63,6 +69,18 @@ resource "google_organization_iam_member" "sa_compute_xpn_admin" {
   member = google_service_account.sa.member
 }
 
+resource "google_organization_iam_member" "sa_contentwarehouse_admin" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/contentwarehouse.admin"
+  member = google_service_account.sa.member
+}
+
+resource "google_organization_iam_member" "sa_contentwarehouse_document_admin" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/contentwarehouse.documentAdmin"
+  member = google_service_account.sa.member
+}
+
 resource "google_organization_iam_member" "sa_deny_admin" {
   org_id = data.google_organization.org.org_id
   role   = "roles/iam.denyAdmin"
@@ -135,12 +153,6 @@ resource "google_organization_iam_member" "sa_storage_admin" {
   member = google_service_account.sa.member
 }
 
-resource "google_organization_iam_member" "apphub_admin" {
-  org_id = data.google_organization.org.org_id
-  role               = "roles/apphub.admin"
-  member             = google_service_account.sa.member
-}
-
 resource "google_billing_account_iam_member" "sa_master_billing_admin" {
   billing_account_id = data.google_billing_account.master_acct.id
   role               = "roles/billing.admin"
@@ -286,6 +298,7 @@ module "project-services" {
     "oslogin.googleapis.com",
     "parallelstore.googleapis.com",
     "privateca.googleapis.com",
+    "privilegedaccessmanager.googleapis.com",
     "pubsub.googleapis.com",
     "pubsublite.googleapis.com",
     "publicca.googleapis.com",

.ci/magician/cmd/check_cassettes.go

@@ -28,7 +28,6 @@ var ccEnvironmentVariables = [...]string{
 	"GOOGLE_REGION",
 	"GOOGLE_SERVICE_ACCOUNT",
 	"GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION",
-	"GOOGLE_TPU_V2_VM_RUNTIME_VERSION",
 	"GOOGLE_ZONE",
 	"PATH",
 	"SA_KEY",

.ci/magician/cmd/test_terraform_vcr.go

@@ -30,7 +30,6 @@ var ttvEnvironmentVariables = [...]string{
 	"GOOGLE_REGION",
 	"GOOGLE_SERVICE_ACCOUNT",
 	"GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION",
-	"GOOGLE_TPU_V2_VM_RUNTIME_VERSION",
 	"GOOGLE_ZONE",
 	"HOME",
 	"PATH",

.github/workflows/request-reviewer.yml

@@ -9,13 +9,13 @@ on:
       - ready_for_review
       - reopened
       - synchronize
-    branches:
-      - 'main'
-      - 'FEATURE-BRANCH-*'
+  issue_comment:
+    types:
+      - created
 
 jobs:
   request-review:
-    if: github.event.pull_request.draft == false
+    if: github.event.pull_request && github.event.pull_request.draft == false && (github.event.sender.login == github.event.pull_request.user.login || github.event.action != 'created')
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write

mmv1/api/resource.go

@@ -132,7 +132,7 @@ type Resource struct {
 	//
 	// [Optional] (Api::Resource::IamPolicy) Configuration of a resource's
 	// resource-specific IAM Policy.
-	IamPolicy resource.IamPolicy `yaml:"iam_policy"`
+	IamPolicy *resource.IamPolicy `yaml:"iam_policy"`
 
 	// [Optional] If set to true, don't generate the resource itself; only
 	// generate the IAM policy.
@@ -964,13 +964,18 @@ func (r Resource) GetIdFormat() string {
 // ====================
 // Template Methods
 // ====================
+// Functions used to create slices of resource properties that could not otherwise be called from within generating templates.
 
-// Prints a dot notation path to where the field is nested within the parent
-// object when called on a property. eg: parent.meta.label.foo
-// Redefined on Resource to terminate the calls up the parent chain.
+func (r Resource) ReadProperties() []*Type {
+	return google.Reject(r.GettableProperties(), func(p *Type) bool {
+		return p.IgnoreRead
+	})
+}
 
-// checks a resource for if it has properties that have FlattenObject=true on fields where IgnoreRead=false
-// used to decide whether or not to import "google.golang.org/api/googleapi"
 func (r Resource) FlattenedProperties() []*Type {
-	return google.Select(google.Reject(r.GettableProperties(), func(p *Type) bool { return p.IgnoreRead }), func(p *Type) bool { return p.FlattenObject })
+	return google.Select(r.ReadProperties(), func(p *Type) bool { 
+		return p.FlattenObject 
+	})
 }
+
+

mmv1/main.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 
 	"golang.org/x/exp/slices"
 
@@ -68,6 +69,7 @@ func main() {
 		log.Fatalf("No product.yaml file found.")
 	}
 
+	startTime := time.Now()
 	log.Printf("Generating MM output to '%s'", *outputPath)
 	log.Printf("Using %s version", *version)
 
@@ -80,6 +82,7 @@ func main() {
 		return false
 	})
 
+	var productsForVersion []map[string]interface{}
 	for _, productName := range allProductFiles {
 		productYamlPath := path.Join(productName, "go_product.yaml")
 
@@ -137,7 +140,7 @@ func main() {
 			productApi.Validate()
 
 			// TODO Q2: set other providers via flag
-			providerToGenerate := provider.NewTerraform(productApi, *version)
+			providerToGenerate := provider.NewTerraform(productApi, *version, startTime)
 
 			if !slices.Contains(productsToGenerate, productName) {
 				log.Printf("%s not specified, skipping generation", productName)
@@ -146,8 +149,33 @@ func main() {
 
 			log.Printf("%s: Generating files", productName)
 			providerToGenerate.Generate(*outputPath, productName, generateCode, generateDocs)
+
+			// we need to preserve a single provider instance to use outside of this loop.
+			productsForVersion = append(productsForVersion, map[string]interface{}{
+				"Definitions": productApi,
+				"Provider":    providerToGenerate,
+			})
 		}
 
 		// TODO Q2: copy common files
 	}
+
+	slices.SortFunc(productsForVersion, func(p1, p2 map[string]interface{}) int {
+		return strings.Compare(strings.ToLower(p1["Definitions"].(*api.Product).Name), strings.ToLower(p2["Definitions"].(*api.Product).Name))
+	})
+
+	// In order to only copy/compile files once per provider this must be called outside
+	// of the products loop. This will get called with the provider from the final iteration
+	// of the loop
+	finalProduct := productsForVersion[len(productsForVersion)-1]
+	provider := finalProduct["Provider"].(*provider.Terraform)
+
+	provider.CopyCommonFiles(*outputPath, generateCode, generateDocs)
+
+	log.Printf("Compiling common files for terraform")
+	if generateCode {
+		provider.CompileCommonFiles(*outputPath, productsForVersion, "")
+
+		// TODO Q2: product overrides
+	}
 }

mmv1/products/alloydb/Cluster.yaml

@@ -497,6 +497,56 @@ properties:
         description: |
           Name of the primary cluster must be in the format
           'projects/{project}/locations/{location}/clusters/{cluster_id}'
+  - !ruby/object:Api::Type::NestedObject
+    name: 'maintenanceUpdatePolicy'
+    description: |
+      MaintenanceUpdatePolicy defines the policy for system updates.
+    properties:
+      - !ruby/object:Api::Type::Array
+        name: 'maintenanceWindows'
+        description: |
+          Preferred windows to perform maintenance. Currently limited to 1.
+        item_type: !ruby/object:Api::Type::NestedObject
+          name: 'maintenanceWindow'
+          description: |
+            specifies a preferred day and time for maintenance.
+          properties:
+            - !ruby/object:Api::Type::Enum
+              name: 'day'
+              required: true
+              description: |
+                Preferred day of the week for maintenance, e.g. MONDAY, TUESDAY, etc.
+              values:
+                - :MONDAY
+                - :TUESDAY
+                - :WEDNESDAY
+                - :THURSDAY
+                - :FRIDAY
+                - :SATURDAY
+                - :SUNDAY
+            - !ruby/object:Api::Type::NestedObject
+              name: 'startTime'
+              required: true
+              description: |
+                Preferred time to start the maintenance operation on the specified day. Maintenance will start within 1 hour of this time.
+              properties:
+                - !ruby/object:Api::Type::Integer
+                  name: hours
+                  required: true
+                  description: |
+                    Hours of day in 24 hour format. Should be from 0 to 23.
+                - !ruby/object:Api::Type::Integer
+                  name: minutes
+                  description: |
+                    Minutes of hour of day. Currently, only the value 0 is supported.
+                - !ruby/object:Api::Type::Integer
+                  name: seconds
+                  description: |
+                    Seconds of minutes of the time. Currently, only the value 0 is supported.
+                - !ruby/object:Api::Type::Integer
+                  name: nanos
+                  description: |
+                    Fractions of seconds in nanoseconds. Currently, only the value 0 is supported.
 virtual_fields:
   - !ruby/object:Api::Type::Enum
     name: 'deletion_policy'

mmv1/products/bigquery/Dataset.yaml

@@ -79,16 +79,14 @@ examples:
       dataset_id: 'example_dataset'
       account_name: 'bqowner'
   - !ruby/object:Provider::Terraform::Examples
-    name: 'bigquery_dataset_external_reference_aws'
+    name: 'bigquery_dataset_external_reference_aws_test'
     primary_resource_id: 'dataset'
-    min_version: beta
     skip_docs: true
     vars:
       dataset_id: 'example_dataset'
   - !ruby/object:Provider::Terraform::Examples
-    name: 'bigquery_dataset_external_reference_aws_docs'
+    name: 'bigquery_dataset_external_reference_aws'
     primary_resource_id: 'dataset'
-    min_version: beta
     skip_test: true
     vars:
       dataset_id: 'example_dataset'
@@ -309,7 +307,6 @@ properties:
     name: 'externalDatasetReference'
     description: |
      Information about the external metadata storage where the dataset is defined.
-    min_version: beta
     immutable: true
     properties:
       - !ruby/object:Api::Type::String

mmv1/products/compute/RegionTargetHttpsProxy.yaml

@@ -58,6 +58,18 @@ examples:
       region_url_map_name: 'url-map'
       region_backend_service_name: 'backend-service'
       region_health_check_name: 'http-health-check'
+  - !ruby/object:Provider::Terraform::Examples
+    name: 'region_target_https_proxy_mtls'
+    primary_resource_id: 'default'
+    min_version: 'beta'
+    vars:
+      target_https_proxy_name: 'test-mtls-proxy'
+      ssl_certificate_name: 'my-certificate'
+      url_map_name: 'url-map'
+      backend_service_name: 'backend-service'
+      http_health_check_name: 'http-health-check'
+      server_tls_policy_name: 'my-tls-policy'
+      trust_config_name: 'my-trust-config'
   - !ruby/object:Provider::Terraform::Examples
     name: 'region_target_https_proxy_certificate_manager_certificate'
     primary_resource_id: 'default'
@@ -179,3 +191,17 @@ properties:
     update_verb: :POST
     custom_expand: 'templates/terraform/custom_expand/resourceref_with_validation.go.erb'
     update_url: 'projects/{{project}}/regions/{{region}}/targetHttpsProxies/{{name}}/setUrlMap'
+  - !ruby/object:Api::Type::ResourceRef
+    name: 'serverTlsPolicy'
+    resource: 'SslPolicy'
+    imports: 'selfLink'
+    description: |
+      A URL referring to a networksecurity.ServerTlsPolicy
+      resource that describes how the proxy should authenticate inbound
+      traffic. serverTlsPolicy only applies to a global TargetHttpsProxy
+      attached to globalForwardingRules with the loadBalancingScheme
+      set to INTERNAL_SELF_MANAGED or EXTERNAL or EXTERNAL_MANAGED.
+      For details which ServerTlsPolicy resources are accepted with
+      INTERNAL_SELF_MANAGED and which with EXTERNAL, EXTERNAL_MANAGED
+      loadBalancingScheme consult ServerTlsPolicy documentation.
+      If left blank, communications are not encrypted.