Go rewrite tgc iam converter template and handwritten test files (Goo…

…gleCloudPlatform#11740)

mmv1/api/resource.go

@@ -1192,8 +1192,7 @@ func (r Resource) ExtractIdentifiers(url string) []string {
 	return result
 }
 
-// For example, "projects/{{project}}/schemas/{{name}}", "{{project}}/{{name}}", "{{name}}"
-func (r Resource) RawImportIdFormatsFromIam() []string {
+func (r Resource) IamImportFormats() []string {
 	var importFormat []string
 
 	if r.IamPolicy != nil {
@@ -1202,8 +1201,12 @@ func (r Resource) RawImportIdFormatsFromIam() []string {
 	if len(importFormat) == 0 {
 		importFormat = r.ImportFormat
 	}
+	return importFormat
+}
 
-	return ImportIdFormats(importFormat, r.Identity, r.BaseUrl)
+// For example, "projects/{{project}}/schemas/{{name}}", "{{project}}/{{name}}", "{{name}}"
+func (r Resource) RawImportIdFormatsFromIam() []string {
+	return ImportIdFormats(r.IamImportFormats(), r.Identity, r.BaseUrl)
 }
 
 // For example, projects/(?P<project>[^/]+)/schemas/(?P<schema>[^/]+)", "(?P<project>[^/]+)/(?P<schema>[^/]+)", "(?P<schema>[^/]+)
@@ -1668,3 +1671,34 @@ func (r Resource) CaiApiVersion(productBackendName, caiProductBaseUrl string) st
 	}
 	return ""
 }
+
+// For example: the uri "projects/{{project}}/schemas/{{name}}"
+// The paramerter is "schema" as "project" is not returned.
+func (r Resource) CaiIamResourceParams() []string {
+	resourceUri := strings.ReplaceAll(r.IamResourceUri(), "{{name}}", fmt.Sprintf("{{%s}}", r.IamParentResourceName()))
+
+	return google.Reject(r.ExtractIdentifiers(resourceUri), func(param string) bool {
+		return param == "project"
+	})
+}
+
+// Gets the Cai IAM asset name template
+// For example: //monitoring.googleapis.com/v3/projects/{{project}}/services/{{service_id}}
+func (r Resource) CaiIamAssetNameTemplate(productBackendName string) string {
+	iamImportFormat := r.IamImportFormats()
+	if len(iamImportFormat) > 0 {
+		name := strings.ReplaceAll(iamImportFormat[0], "{{name}}", fmt.Sprintf("{{%s}}", r.IamParentResourceName()))
+		name = strings.ReplaceAll(name, "%", "")
+		return fmt.Sprintf("//%s.googleapis.com/%s", productBackendName, name)
+	}
+
+	caiBaseUrl := r.CaiBaseUrl
+
+	if caiBaseUrl == "" {
+		caiBaseUrl = r.SelfLink
+	}
+	if caiBaseUrl == "" {
+		caiBaseUrl = r.BaseUrl
+	}
+	return fmt.Sprintf("//%s.googleapis.com/%s/{{%s}}", productBackendName, caiBaseUrl, r.IamParentResourceName())
+}

mmv1/google/slice_utils.go

@@ -39,3 +39,19 @@ func Reject[T any](S []T, test func(T) bool) (ret []T) {
 func Concat[T any](S1 []T, S2 []T) (ret []T) {
 	return append(S1, S2...)
 }
+
+// difference returns the elements in `S1` that aren't in `S2`.
+func Diff(S1, S2 []string) []string {
+	var ret []string
+	mb := make(map[string]bool, len(S2))
+	for _, x := range S2 {
+		mb[x] = true
+	}
+
+	for _, x := range S1 {
+		if _, found := mb[x]; !found {
+			ret = append(ret, x)
+		}
+	}
+	return ret
+}

mmv1/google/template_utils.go

@@ -65,6 +65,7 @@ var TemplateFunctions = template.FuncMap{
 	"join":          strings.Join,
 	"lower":         strings.ToLower,
 	"upper":         strings.ToUpper,
+	"hasSuffix":     strings.HasSuffix,
 	"dict":          wrapMultipleParams,
 	"format2regex":  Format2Regex,
 	"hasPrefix":     strings.HasPrefix,

mmv1/provider/template_data.go

@@ -177,6 +177,14 @@ func (td *TemplateData) GenerateTGCResourceFile(filePath string, resource api.Re
 	td.GenerateFile(filePath, templatePath, resource, true, templates...)
 }
 
+func (td *TemplateData) GenerateTGCIamResourceFile(filePath string, resource api.Resource) {
+	templatePath := "templates/tgc/resource_converter_iam.go.tmpl"
+	templates := []string{
+		templatePath,
+	}
+	td.GenerateFile(filePath, templatePath, resource, true, templates...)
+}
+
 func (td *TemplateData) GenerateFile(filePath, templatePath string, input any, goFormat bool, templates ...string) {
 	// log.Printf("Generating %s", filePath)
 

mmv1/provider/terraform_tgc.go

@@ -17,18 +17,28 @@ package provider
 
 import (
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"path"
+	"path/filepath"
+	"regexp"
+	"strings"
 	"time"
 
+	"golang.org/x/exp/slices"
+
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/api/product"
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/google"
 )
 
 type TerraformGoogleConversion struct {
-	ResourcesForVersion []map[string]string
+	IamResources []map[string]string
+
+	NonDefinedTests []string
+
+	Tests []string
 
 	TargetVersionName string
 
@@ -56,11 +66,6 @@ func NewTerraformGoogleConversion(product *api.Product, versionName string, star
 	return t
 }
 
-func (tgc TerraformGoogleConversion) generatingHashicorpRepo() bool {
-	// This code is not used when generating TPG/TPGB
-	return false
-}
-
 func (tgc TerraformGoogleConversion) Generate(outputFolder, productPath, resourceToGenerate string, generateCode, generateDocs bool) {
 	// Temporary shim to generate the missing resources directory. Can be removed
 	// once the folder exists downstream.
@@ -106,7 +111,7 @@ func (tgc TerraformGoogleConversion) GenerateObject(object api.Resource, outputF
 		return
 	}
 
-	// tgc.GenerateIamPolicy(object, *templateData, outputFolder, generateCode, generateDocs)
+	tgc.GenerateIamPolicy(object, *templateData, outputFolder, generateCode, generateDocs)
 }
 
 func (tgc TerraformGoogleConversion) GenerateResource(object api.Resource, templateData TemplateData, outputFolder string, generateCode, generateDocs bool) {
@@ -120,8 +125,216 @@ func (tgc TerraformGoogleConversion) GenerateResource(object api.Resource, templ
 	templateData.GenerateTGCResourceFile(targetFilePath, object)
 }
 
+// Generate the IAM policy for this object. This is used to query and test
+// IAM policies separately from the resource itself
+// Docs are generated for the terraform provider, not here.
+func (tgc TerraformGoogleConversion) GenerateIamPolicy(object api.Resource, templateData TemplateData, outputFolder string, generateCode, generateDocs bool) {
+	if !generateCode || object.IamPolicy.ExcludeTgc {
+		return
+	}
+
+	productName := tgc.Product.ApiName
+	targetFolder := path.Join(outputFolder, "converters/google/resources/services", productName)
+	if err := os.MkdirAll(targetFolder, os.ModePerm); err != nil {
+		log.Println(fmt.Errorf("error creating parent directory %v: %v", targetFolder, err))
+	}
+
+	name := object.FilenameOverride
+	if name == "" {
+		name = google.Underscore(object.Name)
+	}
+
+	targetFilePath := path.Join(targetFolder, fmt.Sprintf("%s_%s_iam.go", productName, name))
+	templateData.GenerateTGCIamResourceFile(targetFilePath, object)
+
+	targetFilePath = path.Join(targetFolder, fmt.Sprintf("iam_%s_%s.go", productName, name))
+	templateData.GenerateIamPolicyFile(targetFilePath, object)
+
+	// Don't generate tests - we can rely on the terraform provider
+	//  to test these.
+}
+
+// Generates the list of resources
+func (tgc TerraformGoogleConversion) generateCaiIamResources(products []*api.Product) {
+	for _, productDefinition := range products {
+		service := strings.ToLower(productDefinition.Name)
+		for _, object := range productDefinition.Objects {
+			if object.MinVersionObj().Name != "ga" || object.Exclude || object.ExcludeTgc {
+				continue
+			}
+
+			var iamClassName string
+			iamPolicy := object.IamPolicy
+			if iamPolicy != nil && !iamPolicy.Exclude && !iamPolicy.ExcludeTgc {
+
+				iamClassName = fmt.Sprintf("%s.ResourceConverter%s", service, object.ResourceName())
+
+				tgc.IamResources = append(tgc.IamResources, map[string]string{
+					"TerraformName": object.TerraformName(),
+					"IamClassName":  iamClassName,
+				})
+			}
+		}
+	}
+}
+
 func (tgc TerraformGoogleConversion) CompileCommonFiles(outputFolder string, products []*api.Product, overridePath string) {
+	log.Printf("Compiling common files.")
+
+	tgc.generateCaiIamResources(products)
+	tgc.NonDefinedTests = retrieveFullManifestOfNonDefinedTests()
+
+	files := retrieveFullListOfTestFiles()
+	for _, file := range files {
+		tgc.Tests = append(tgc.Tests, strings.Split(file, ".")[0])
+	}
+	tgc.Tests = slices.Compact(tgc.Tests)
 
+	testSource := make(map[string]string)
+	for target, source := range retrieveTestSourceCodeWithLocation(".tmpl") {
+		target := strings.Replace(target, "go.tmpl", "go", 1)
+		testSource[target] = source
+	}
+
+	templateData := NewTemplateData(outputFolder, tgc.TargetVersionName)
+	tgc.CompileFileList(outputFolder, testSource, *templateData, products)
+
+	// compile_file_list(
+	//   output_folder,
+	//   [
+	// 	['converters/google/resources/services/compute/compute_instance_helpers.go',
+	// 	 'third_party/terraform/services/compute/compute_instance_helpers.go.erb'],
+	// 	['converters/google/resources/resource_converters.go',
+	// 	 'templates/tgc/resource_converters.go.erb'],
+	// 	['converters/google/resources/services/kms/iam_kms_key_ring.go',
+	// 	 'third_party/terraform/services/kms/iam_kms_key_ring.go.erb'],
+	// 	['converters/google/resources/services/kms/iam_kms_crypto_key.go',
+	// 	 'third_party/terraform/services/kms/iam_kms_crypto_key.go.erb'],
+	// 	['converters/google/resources/services/compute/metadata.go',
+	// 	 'third_party/terraform/services/compute/metadata.go.erb'],
+	// 	['converters/google/resources/services/compute/compute_instance.go',
+	// 	 'third_party/tgc/compute_instance.go.erb']
+	//   ],
+	//   file_template
+	// )
+}
+
+func (tgc TerraformGoogleConversion) CompileFileList(outputFolder string, files map[string]string, fileTemplate TemplateData, products []*api.Product) {
+	if err := os.MkdirAll(outputFolder, os.ModePerm); err != nil {
+		log.Println(fmt.Errorf("error creating output directory %v: %v", outputFolder, err))
+	}
+
+	for target, source := range files {
+		targetFile := filepath.Join(outputFolder, target)
+		targetDir := filepath.Dir(targetFile)
+		if err := os.MkdirAll(targetDir, os.ModePerm); err != nil {
+			log.Println(fmt.Errorf("error creating output directory %v: %v", targetDir, err))
+		}
+
+		templates := []string{
+			source,
+		}
+
+		formatFile := filepath.Ext(targetFile) == ".go"
+
+		fileTemplate.GenerateFile(targetFile, source, tgc, formatFile, templates...)
+		// tgc.replaceImportPath(outputFolder, target)
+	}
+}
+
+func retrieveFullManifestOfNonDefinedTests() []string {
+	var tests []string
+	fileMap := make(map[string]bool)
+
+	files := retrieveFullListOfTestFiles()
+	for _, file := range files {
+		tests = append(tests, strings.Split(file, ".")[0])
+		fileMap[file] = true
+	}
+	tests = slices.Compact(tests)
+
+	nonDefinedTests := google.Diff(tests, retrieveListOfManuallyDefinedTests())
+	nonDefinedTests = google.Reject(nonDefinedTests, func(file string) bool {
+		return strings.HasSuffix(file, "_without_default_project")
+	})
+
+	for _, test := range nonDefinedTests {
+		_, ok := fileMap[fmt.Sprintf("%s.json", test)]
+		if !ok {
+			log.Fatalf("test file named %s.json expected but found none", test)
+		}
+
+		_, ok = fileMap[fmt.Sprintf("%s.tf", test)]
+		if !ok {
+			log.Fatalf("test file named %s.tf expected but found none", test)
+		}
+	}
+
+	return nonDefinedTests
+}
+
+// Gets all of the test files in the folder third_party/tgc/tests/data
+func retrieveFullListOfTestFiles() []string {
+	var testFiles []string
+
+	files, err := ioutil.ReadDir("third_party/tgc/tests/data")
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, file := range files {
+		testFiles = append(testFiles, file.Name())
+	}
+	slices.Sort(testFiles)
+
+	return testFiles
+}
+
+func retrieveTestSourceCodeWithLocation(suffix string) map[string]string {
+	var fileNames []string
+	path := "third_party/tgc/tests/source/go"
+	files, err := ioutil.ReadDir(path)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	for _, file := range files {
+		log.Printf("ext %s", filepath.Ext(file.Name()))
+		if filepath.Ext(file.Name()) == suffix {
+			fileNames = append(fileNames, file.Name())
+		}
+	}
+
+	slices.Sort(fileNames)
+
+	testSource := make(map[string]string)
+	for _, file := range fileNames {
+		target := fmt.Sprintf("test/%s", file)
+		source := fmt.Sprintf("%s/%s", path, file)
+		testSource[target] = source
+	}
+	return testSource
+}
+
+func retrieveListOfManuallyDefinedTests() []string {
+	m1 := retrieveListOfManuallyDefinedTestsFromFile("third_party/tgc/tests/source/go/cli_test.go.tmpl")
+	m2 := retrieveListOfManuallyDefinedTestsFromFile("third_party/tgc/tests/source/go/read_test.go.tmpl")
+	return google.Concat(m1, m2)
+}
+
+// Reads the content of the file and then finds all of the tests in the contents
+func retrieveListOfManuallyDefinedTestsFromFile(file string) []string {
+	data, err := os.ReadFile(file)
+	if err != nil {
+		log.Fatalf("Cannot open the file: %v", file)
+	}
+
+	var tests []string
+	testsReg := regexp.MustCompile(`\s*name\s*:\s*"([^,]+)"`)
+	matches := testsReg.FindAllStringSubmatch(string(data), -1)
+	for _, testWithName := range matches {
+		tests = append(tests, testWithName[1])
+	}
+	return tests
 }
 
 func (tgc TerraformGoogleConversion) CopyCommonFiles(outputFolder string, generateCode, generateDocs bool) {

mmv1/templates/terraform/iam_policy.go.tmpl

@@ -12,10 +12,10 @@
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License. */ -}}
-{{/* <% if hc_downstream */}}
+{{- if ne $.Compiler "terraformgoogleconversion-codegen" }}
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: MPL-2.0
-
+{{- end }}
 // ----------------------------------------------------------------------------
 //
 //     ***     AUTO GENERATED CODE    ***    Type: MMv1     ***

mmv1/templates/terraform/operation.go.tmpl

@@ -1,7 +1,7 @@
-{{/* TODO rewrite: if hc_downstream */ -}}
+{{- if ne $.Compiler "terraformgoogleconversion-codegen" }}
 // Copyright (c) HashiCorp, Inc.
 // SPDX-License-Identifier: MPL-2.0
-
+{{- end }}
 // ----------------------------------------------------------------------------
 //
 //     ***     AUTO GENERATED CODE    ***    Type: MMv1     ***