sync-main-FEATURE-BRANCH-ephemeral-resources #1 (#11822)

Signed-off-by: drfaust92 <[email protected]>
Co-authored-by: karolgorc <[email protected]>
Co-authored-by: Augusto Hidalgo <[email protected]>
Co-authored-by: Thomas Rodgers <[email protected]>
Co-authored-by: Nick Elliot <[email protected]>
Co-authored-by: Ilia Lazebnik <[email protected]>
Co-authored-by: vijaykanthm <[email protected]>
Co-authored-by: Abhijeet Jha <[email protected]>
Co-authored-by: bcreddy-gcp <[email protected]>
Co-authored-by: Cameron Thornton <[email protected]>
Co-authored-by: Riley Karson <[email protected]>
Co-authored-by: Zhenhua Li <[email protected]>
Co-authored-by: Ron Gal <[email protected]>

GNUmakefile

@@ -9,9 +9,7 @@ ifeq ($(ENGINE),tpgtools)
   # exist so exclusively build base tpgtools implementation
   mmv1_compile=-p does-not-exist
 else ifneq ($(PRODUCT),)
-  mmv1_compile=-p products/$(PRODUCT)
-else
-  mmv1_compile=-a
+  mmv1_compile=--product $(PRODUCT)
 endif
 
 # tpgtools setup
@@ -26,12 +24,12 @@ else
 endif
 
 ifneq ($(RESOURCE),)
-  mmv1_compile += -t $(RESOURCE)
+  mmv1_compile += --resource $(RESOURCE)
   tpgtools_compile += --resource $(RESOURCE)
 endif
 
 ifneq ($(OVERRIDES),)
-  mmv1_compile += -r $(OVERRIDES)
+  mmv1_compile += --overrides $(OVERRIDES)
   tpgtools_compile += --overrides $(OVERRIDES)/tpgtools/overrides --path $(OVERRIDES)/tpgtools/api
   serialize_compile = --overrides $(OVERRIDES)/tpgtools/overrides --path $(OVERRIDES)/tpgtools/api
 else
@@ -62,12 +60,11 @@ terraform build provider:
 
 mmv1:
 	cd mmv1;\
-		bundle; \
 		if [ "$(VERSION)" = "ga" ]; then \
-			bundle exec compiler.rb -e terraform -o $(OUTPUT_PATH) -v ga --no-docs $(mmv1_compile); \
-			bundle exec compiler.rb -e terraform -o $(OUTPUT_PATH) -v beta --no-code $(mmv1_compile); \
+			go run . --output $(OUTPUT_PATH) --version ga --no-docs $(mmv1_compile); \
+			go run . --output $(OUTPUT_PATH) --version beta --no-code $(mmv1_compile); \
 		else \
-			bundle exec compiler.rb -e terraform -o $(OUTPUT_PATH) -v $(VERSION) $(mmv1_compile); \
+			go run . --output $(OUTPUT_PATH) --version $(VERSION) $(mmv1_compile); \
 		fi
 
 tpgtools:
@@ -91,24 +88,16 @@ clean-tgc:
 
 tgc:
 	cd mmv1;\
-		bundle;\
-		bundle exec compiler -e terraform -f tgc -v beta -o $(OUTPUT_PATH)/tfplan2cai $(mmv1_compile);\
-		bundle exec compiler -e terraform -f tgc_cai2hcl -v beta -o $(OUTPUT_PATH)/cai2hcl $(mmv1_compile);\
-
-tgc-go:
-	cd mmv1;\
-		go run . --version beta --provider  tgc --output $(OUTPUT_PATH)/tfplan2cai;\
-		go run . --version beta --provider  tgc_cai2hcl --output $(OUTPUT_PATH)/cai2hcl;\
+		go run . --version beta --provider tgc --output $(OUTPUT_PATH)/tfplan2cai $(mmv1_compile);\
+		go run . --version beta --provider tgc_cai2hcl --output $(OUTPUT_PATH)/cai2hcl $(mmv1_compile);\
 
 tf-oics:
 	cd mmv1;\
-		bundle;\
-		bundle exec compiler.rb -e terraform -f oics -o $(OUTPUT_PATH) $(mmv1_compile);\
+		go run . --version ga --provider oics --output $(OUTPUT_PATH) $(mmv1_compile);\
 
 test:
 	cd mmv1; \
-		bundle; \
-		bundle exec rake test
+		go test ./...
 
 serialize:
 	cd tpgtools;\

mmv1/api/resource.go

@@ -42,8 +42,8 @@ type Resource struct {
 	//
 	//	references:
 	//  	guides:
-    //			'Guide name': 'official_documentation_url'
-  	//		api: 'rest_api_reference_url/version'
+	//			'Guide name': 'official_documentation_url'
+	//		api: 'rest_api_reference_url/version'
 	//
 	References resource.ReferenceLinks
 
@@ -190,7 +190,7 @@ type Resource struct {
 	//
 	//	import_format:
 	//		- example_import_one
-    //		- example_import_two
+	//		- example_import_two
 	//
 	ImportFormat []string `yaml:"import_format"`
 
@@ -760,7 +760,7 @@ func getLabelsFieldNote(title string) string {
 }
 
 func (r Resource) StateMigrationFile() string {
-	return fmt.Sprintf("templates/terraform/state_migrations/go/%s_%s.go.tmpl", google.Underscore(r.ProductMetadata.Name), google.Underscore(r.Name))
+	return fmt.Sprintf("templates/terraform/state_migrations/%s_%s.go.tmpl", google.Underscore(r.ProductMetadata.Name), google.Underscore(r.Name))
 }
 
 // ====================

mmv1/api/resource/examples.go

@@ -170,7 +170,7 @@ func (e *Examples) UnmarshalYAML(unmarshal func(any) error) error {
 	}
 
 	if e.ConfigPath == "" {
-		e.ConfigPath = fmt.Sprintf("templates/terraform/examples/go/%s.tf.tmpl", e.Name)
+		e.ConfigPath = fmt.Sprintf("templates/terraform/examples/%s.tf.tmpl", e.Name)
 	}
 	e.SetHCLText()
 
@@ -286,7 +286,7 @@ func ExecuteTemplate(e any, templatePath string, appendNewline bool) string {
 	templates := []string{
 		templatePath,
 		"templates/terraform/expand_resource_ref.tmpl",
-		"templates/terraform/custom_flatten/go/bigquery_table_ref.go.tmpl",
+		"templates/terraform/custom_flatten/bigquery_table_ref.go.tmpl",
 		"templates/terraform/flatten_property_method.go.tmpl",
 		"templates/terraform/expand_property_method.go.tmpl",
 		"templates/terraform/update_mask.go.tmpl",

mmv1/api/resource/iam_policy.go

@@ -125,7 +125,7 @@ func (p *IamPolicy) UnmarshalYAML(unmarshal func(any) error) error {
 	p.WrappedPolicyObj = true
 	p.AllowedIamRole = "roles/viewer"
 	p.ParentResourceAttribute = "id"
-	p.ExampleConfigBody = "templates/terraform/iam/go/iam_attributes.go.tmpl"
+	p.ExampleConfigBody = "templates/terraform/iam/iam_attributes.go.tmpl"
 	p.SubstituteZoneValue = true
 
 	type iamPolicyAlias IamPolicy

mmv1/main.go

@@ -158,20 +158,27 @@ func main() {
 	})
 
 	var providerToGenerate provider.Provider
-	var productsForVersion []*api.Product
 
-	ch := make(chan string, len(allProductFiles))
+	productFileChannel := make(chan string, len(allProductFiles))
+	productsForVersionChannel := make(chan *api.Product, len(allProductFiles))
 	for _, pf := range allProductFiles {
-		ch <- pf
+		productFileChannel <- pf
 	}
 
 	for i := 0; i < len(allProductFiles); i++ {
 		wg.Add(1)
-		go GenerateProduct(ch, providerToGenerate, &productsForVersion, startTime, productsToGenerate, *resourceToGenerate, *overrideDirectory, generateCode, generateDocs)
+		go GenerateProduct(productFileChannel, providerToGenerate, productsForVersionChannel, startTime, productsToGenerate, *resourceToGenerate, *overrideDirectory, generateCode, generateDocs)
 	}
 	wg.Wait()
 
-	close(ch)
+	close(productFileChannel)
+	close(productsForVersionChannel)
+
+	var productsForVersion []*api.Product
+
+	for p := range productsForVersionChannel {
+		productsForVersion = append(productsForVersion, p)
+	}
 
 	slices.SortFunc(productsForVersion, func(p1, p2 *api.Product) int {
 		return strings.Compare(strings.ToLower(p1.Name), strings.ToLower(p2.Name))
@@ -185,21 +192,19 @@ func main() {
 
 	if generateCode {
 		providerToGenerate.CompileCommonFiles(*outputPath, productsForVersion, "")
-
-		// TODO rewrite: product overrides
 	}
 }
 
-func GenerateProduct(productChannel chan string, providerToGenerate provider.Provider, productsForVersion *[]*api.Product, startTime time.Time, productsToGenerate []string, resourceToGenerate, overrideDirectory string, generateCode, generateDocs bool) {
+func GenerateProduct(productChannel chan string, providerToGenerate provider.Provider, productsForVersionChannel chan *api.Product, startTime time.Time, productsToGenerate []string, resourceToGenerate, overrideDirectory string, generateCode, generateDocs bool) {
 
 	defer wg.Done()
 	productName := <-productChannel
 
-	productYamlPath := path.Join(productName, "go_product.yaml")
+	productYamlPath := path.Join(productName, "product.yaml")
 
 	var productOverridePath string
 	if overrideDirectory != "" {
-		productOverridePath = filepath.Join(overrideDirectory, productName, "go_product.yaml")
+		productOverridePath = filepath.Join(overrideDirectory, productName, "product.yaml")
 	}
 
 	_, baseProductErr := os.Stat(productYamlPath)
@@ -245,11 +250,6 @@ func GenerateProduct(productChannel chan string, providerToGenerate provider.Pro
 			continue
 		}
 
-		// Prepend "go_" to the Go yaml files' name to distinguish with the ruby yaml files
-		if filepath.Base(resourceYamlPath) == "go_product.yaml" || !strings.HasPrefix(filepath.Base(resourceYamlPath), "go_") {
-			continue
-		}
-
 		if overrideDirectory != "" {
 			// skip if resource will be merged in the override loop
 			resourceOverridePath := filepath.Join(overrideDirectory, resourceYamlPath)
@@ -308,8 +308,6 @@ func GenerateProduct(productChannel chan string, providerToGenerate provider.Pro
 			resources = append(resources, resource)
 		}
 
-		// TODO rewrite: override resources
-
 		// Sort resources by name
 		sort.Slice(resources, func(i, j int) bool {
 			return resources[i].Name < resources[j].Name
@@ -322,7 +320,7 @@ func GenerateProduct(productChannel chan string, providerToGenerate provider.Pro
 
 	providerToGenerate = setProvider(*forceProvider, *version, productApi, startTime)
 
-	*productsForVersion = append(*productsForVersion, productApi)
+	productsForVersionChannel <- productApi
 
 	if !slices.Contains(productsToGenerate, productName) {
 		log.Printf("%s not specified, skipping generation", productName)

mmv1/products/accessapproval/FolderSettings.yaml

@@ -1,4 +1,4 @@
-# Copyright 2023 Google Inc.
+# Copyright 2024 Google Inc.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -11,80 +11,86 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
---- !ruby/object:Api::Resource
-name: FolderSettings
-base_url: 'folders/{{folder_id}}/accessApprovalSettings'
-self_link: 'folders/{{folder_id}}/accessApprovalSettings'
-create_verb: :PATCH
-update_verb: :PATCH
-update_mask: true
+---
+name: 'FolderSettings'
+legacy_name: 'google_folder_access_approval_settings'
 description: |
   Access Approval enables you to require your explicit approval whenever Google support and engineering need to access your customer content.
-references: !ruby/object:Api::Resource::ReferenceLinks
+references:
+  guides:
   api: 'https://cloud.google.com/access-approval/docs/reference/rest/v1/folders'
-legacy_name: 'google_folder_access_approval_settings'
-import_format: ['folders/{{folder_id}}/accessApprovalSettings']
+docs:
+base_url: 'folders/{{folder_id}}/accessApprovalSettings'
+self_link: 'folders/{{folder_id}}/accessApprovalSettings'
+create_verb: 'PATCH'
+update_verb: 'PATCH'
+update_mask: true
+import_format:
+  - 'folders/{{folder_id}}/accessApprovalSettings'
+timeouts:
+  insert_minutes: 20
+  update_minutes: 20
+  delete_minutes: 20
+custom_code:
+  constants: 'templates/terraform/constants/access_approval.go.tmpl'
+  pre_create: 'templates/terraform/update_mask.go.tmpl'
+  custom_delete: 'templates/terraform/custom_delete/clear_folder_access_approval_settings.go.tmpl'
 examples:
-  - !ruby/object:Provider::Terraform::Examples
-    skip_test: true
-    name: 'folder_access_approval_full'
+  - name: 'folder_access_approval_full'
     primary_resource_id: 'folder_access_approval'
     vars:
       folder_name: 'my-folder'
     test_env_vars:
-      org_id: :ORG_ID
-  - !ruby/object:Provider::Terraform::Examples
-    skip_test: true
-    name: 'folder_access_approval_active_key_version'
+      org_id: 'ORG_ID'
+    exclude_test: true
+  - name: 'folder_access_approval_active_key_version'
     primary_resource_id: 'folder_access_approval'
     vars:
       folder_name: 'my-folder'
     test_env_vars:
-      org_id: :ORG_ID
-custom_code: !ruby/object:Provider::Terraform::CustomCode
-  custom_delete: templates/terraform/custom_delete/clear_folder_access_approval_settings.go.erb
-  pre_create: templates/terraform/update_mask.erb
-  constants: templates/terraform/constants/access_approval.go.erb
+      org_id: 'ORG_ID'
+    exclude_test: true
 parameters:
-  - !ruby/object:Api::Type::String
-    name: folder_id
+  - name: 'folder_id'
+    type: String
     description: |
       ID of the folder of the access approval settings.
+    url_param_only: true
     required: true
     immutable: true
-    url_param_only: true
 properties:
-  - !ruby/object:Api::Type::String
-    name: name
+  - name: 'name'
+    type: String
     description: |
       The resource name of the settings. Format is "folders/{folder_id}/accessApprovalSettings"
     output: true
-  - !ruby/object:Api::Type::Array
-    name: notificationEmails
+  - name: 'notificationEmails'
+    type: Array
     description: |
       A list of email addresses to which notifications relating to approval requests should be sent.
       Notifications relating to a resource will be sent to all emails in the settings of ancestor
       resources of that resource. A maximum of 50 email addresses are allowed.
-    max_size: 50
-    item_type: Api::Type::String
     is_set: true
     default_from_api: true
-  - !ruby/object:Api::Type::Array
-    name: enrolledServices
-    required: true
+    item_type:
+      type: String
+    max_size: 50
+  - name: 'enrolledServices'
+    type: Array
     description: |
       A list of Google Cloud Services for which the given resource has Access Approval enrolled.
       Access requests for the resource given by name against any of these services contained here will be required
       to have explicit approval. Enrollment can only be done on an all or nothing basis.
 
       A maximum of 10 enrolled services will be enforced, to be expanded as the set of supported services is expanded.
     is_set: true
+    required: true
     set_hash_func: accessApprovalEnrolledServicesHash
-    item_type: !ruby/object:Api::Type::NestedObject
+    item_type:
+      type: NestedObject
       properties:
-        - !ruby/object:Api::Type::String
-          name: cloudProduct
-          required: true
+        - name: 'cloudProduct'
+          type: String
           description: |
             The product for which Access Approval will be enrolled. Allowed values are listed (case-sensitive):
               * all
@@ -110,34 +116,35 @@ properties:
               * iam.googleapis.com
               * pubsub.googleapis.com
               * storage.googleapis.com
-        - !ruby/object:Api::Type::Enum
-          name: enrollmentLevel
+          required: true
+        - name: 'enrollmentLevel'
+          type: Enum
           description: |
             The enrollment level of the service.
-          default_value: :BLOCK_ALL
-          values:
-            - :BLOCK_ALL
-  - !ruby/object:Api::Type::Boolean
-    name: enrolledAncestor
-    output: true
+          default_value: "BLOCK_ALL"
+          enum_values:
+            - 'BLOCK_ALL'
+  - name: 'enrolledAncestor'
+    type: Boolean
     description: |
       If the field is true, that indicates that at least one service is enrolled for Access Approval in one or more ancestors of the Folder.
-  - !ruby/object:Api::Type::String
-    name: activeKeyVersion
+    output: true
+  - name: 'activeKeyVersion'
+    type: String
     description: |
       The asymmetric crypto key version to use for signing approval requests.
       Empty active_key_version indicates that a Google-managed key should be used for signing.
       This property will be ignored if set by an ancestor of the resource, and new non-empty values may not be set.
-  - !ruby/object:Api::Type::Boolean
-    name: ancestorHasActiveKeyVersion
-    output: true
+  - name: 'ancestorHasActiveKeyVersion'
+    type: Boolean
     description: |
       If the field is true, that indicates that an ancestor of this Folder has set active_key_version.
-  - !ruby/object:Api::Type::Boolean
-    name: invalidKeyVersion
     output: true
+  - name: 'invalidKeyVersion'
+    type: Boolean
     description: |
       If the field is true, that indicates that there is some configuration issue with the active_key_version
       configured on this Folder (e.g. it doesn't exist or the Access Approval service account doesn't have the
       correct permissions on it, etc.) This key version is not necessarily the effective key version at this level,
       as key versions are inherited top-down.
+    output: true