go rewrite - initial concurrency (GoogleCloudPlatform#11627)

BBBmau · Sep 3, 2024 · b39a08d · b39a08d
1 parent 5cdc1d8
commit b39a08d
Show file tree

Hide file tree

Showing 2 changed files with 88 additions and 65 deletions.
diff --git a/mmv1/main.go b/mmv1/main.go
@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"golang.org/x/exp/slices"
@@ -17,6 +18,8 @@ import (
 	"github.com/GoogleCloudPlatform/magic-modules/mmv1/provider"
 )
 
+var wg sync.WaitGroup
+
 // TODO Q2: additional flags
 
 // Example usage: --output $GOPATH/src/github.com/terraform-providers/terraform-provider-google-beta
@@ -100,98 +103,118 @@ func main() {
 	// Building compute takes a long time and can't be parallelized within the product
 	// so lets build it first
 	sort.Slice(allProductFiles, func(i int, j int) bool {
-		if allProductFiles[i] == "compute" {
+		if allProductFiles[i] == "products/compute" {
 			return true
 		}
 		return false
 	})
 
+	var providerToGenerate *provider.Terraform
+	var productsForVersion []*api.Product
+
+	ch := make(chan string, len(allProductFiles))
+	for _, pf := range allProductFiles {
+		ch <- pf
+	}
+
+	for i := 0; i < len(allProductFiles); i++ {
+		wg.Add(1)
+		go GenerateProduct(ch, providerToGenerate, &productsForVersion, startTime, productsToGenerate, generateCode, generateDocs)
+	}
+	wg.Wait()
+
+	close(ch)
+
+	slices.SortFunc(productsForVersion, func(p1, p2 *api.Product) int {
+		return strings.Compare(strings.ToLower(p1.Name), strings.ToLower(p2.Name))
+	})
+
 	// In order to only copy/compile files once per provider this must be called outside
 	// of the products loop. This will get called with the provider from the final iteration
 	// of the loop
-	var providerToGenerate *provider.Terraform
-	var productsForVersion []*api.Product
-	for _, productName := range allProductFiles {
-		productYamlPath := path.Join(productName, "go_product.yaml")
+	providerToGenerate = provider.NewTerraform(productsForVersion[0], *version, startTime)
 
-		// TODO Q2: uncomment the error check that if the product.yaml exists for each product
-		// after Go-converted product.yaml files are complete for all products
-		// if _, err := os.Stat(productYamlPath); errors.Is(err, os.ErrNotExist) {
-		// 	log.Fatalf("%s does not contain a product.yaml file", productName)
-		// }
+	providerToGenerate.CopyCommonFiles(*outputPath, generateCode, generateDocs)
+
+	log.Printf("Compiling common files for terraform")
+	if generateCode {
+		providerToGenerate.CompileCommonFiles(*outputPath, productsForVersion, "")
 
 		// TODO Q2: product overrides
+	}
+}
 
-		if _, err := os.Stat(productYamlPath); err == nil {
-			var resources []*api.Resource = make([]*api.Resource, 0)
+func GenerateProduct(productChannel chan string, providerToGenerate *provider.Terraform, productsForVersion *[]*api.Product, startTime time.Time, productsToGenerate []string, generateCode, generateDocs bool) {
 
-			productApi := &api.Product{}
-			api.Compile(productYamlPath, productApi)
+	defer wg.Done()
+	productName := <-productChannel
 
-			if !productApi.ExistsAtVersionOrLower(*version) {
-				log.Printf("%s does not have a '%s' version, skipping", productName, *version)
-				continue
-			}
+	productYamlPath := path.Join(productName, "go_product.yaml")
 
-			resourceFiles, err := filepath.Glob(fmt.Sprintf("%s/*", productName))
-			if err != nil {
-				log.Fatalf("Cannot get resources files: %v", err)
-			}
-			for _, resourceYamlPath := range resourceFiles {
-				if filepath.Base(resourceYamlPath) == "product.yaml" || filepath.Ext(resourceYamlPath) != ".yaml" {
-					continue
-				}
-
-				// Prepend "go_" to the Go yaml files' name to distinguish with the ruby yaml files
-				if filepath.Base(resourceYamlPath) == "go_product.yaml" || !strings.HasPrefix(filepath.Base(resourceYamlPath), "go_") {
-					continue
-				}
-
-				resource := &api.Resource{}
-				api.Compile(resourceYamlPath, resource)
-
-				resource.TargetVersionName = *version
-				resource.Properties = resource.AddLabelsRelatedFields(resource.PropertiesWithExcluded(), nil)
-				resource.SetDefault(productApi)
-				resource.Validate()
-				resources = append(resources, resource)
-			}
+	// TODO Q2: uncomment the error check that if the product.yaml exists for each product
+	// after Go-converted product.yaml files are complete for all products
+	// if _, err := os.Stat(productYamlPath); errors.Is(err, os.ErrNotExist) {
+	// 	log.Fatalf("%s does not contain a product.yaml file", productName)
+	// }
 
-			// TODO Q2: override resources
+	// TODO Q2: product overrides
 
-			// Sort resources by name
-			sort.Slice(resources, func(i, j int) bool {
-				return resources[i].Name < resources[j].Name
-			})
+	if _, err := os.Stat(productYamlPath); err == nil {
+		var resources []*api.Resource = make([]*api.Resource, 0)
 
-			productApi.Objects = resources
-			productApi.Validate()
+		productApi := &api.Product{}
+		api.Compile(productYamlPath, productApi)
 
-			// TODO Q2: set other providers via flag
-			providerToGenerate = provider.NewTerraform(productApi, *version, startTime)
+		if !productApi.ExistsAtVersionOrLower(*version) {
+			log.Printf("%s does not have a '%s' version, skipping", productName, *version)
+			return
+		}
 
-			productsForVersion = append(productsForVersion, productApi)
+		resourceFiles, err := filepath.Glob(fmt.Sprintf("%s/*", productName))
+		if err != nil {
+			log.Fatalf("Cannot get resources files: %v", err)
+		}
+		for _, resourceYamlPath := range resourceFiles {
+			if filepath.Base(resourceYamlPath) == "product.yaml" || filepath.Ext(resourceYamlPath) != ".yaml" {
+				continue
+			}
 
-			if !slices.Contains(productsToGenerate, productName) {
-				log.Printf("%s not specified, skipping generation", productName)
+			// Prepend "go_" to the Go yaml files' name to distinguish with the ruby yaml files
+			if filepath.Base(resourceYamlPath) == "go_product.yaml" || !strings.HasPrefix(filepath.Base(resourceYamlPath), "go_") {
 				continue
 			}
 
-			log.Printf("%s: Generating files", productName)
-			providerToGenerate.Generate(*outputPath, productName, generateCode, generateDocs)
+			resource := &api.Resource{}
+			api.Compile(resourceYamlPath, resource)
+
+			resource.TargetVersionName = *version
+			resource.Properties = resource.AddLabelsRelatedFields(resource.PropertiesWithExcluded(), nil)
+			resource.SetDefault(productApi)
+			resource.Validate()
+			resources = append(resources, resource)
 		}
-	}
 
-	slices.SortFunc(productsForVersion, func(p1, p2 *api.Product) int {
-		return strings.Compare(strings.ToLower(p1.Name), strings.ToLower(p2.Name))
-	})
+		// TODO Q2: override resources
 
-	providerToGenerate.CopyCommonFiles(*outputPath, generateCode, generateDocs)
+		// Sort resources by name
+		sort.Slice(resources, func(i, j int) bool {
+			return resources[i].Name < resources[j].Name
+		})
 
-	log.Printf("Compiling common files for terraform")
-	if generateCode {
-		providerToGenerate.CompileCommonFiles(*outputPath, productsForVersion, "")
+		productApi.Objects = resources
+		productApi.Validate()
 
-		// TODO Q2: product overrides
+		// TODO Q2: set other providers via flag
+		providerToGenerate = provider.NewTerraform(productApi, *version, startTime)
+
+		*productsForVersion = append(*productsForVersion, productApi)
+
+		if !slices.Contains(productsToGenerate, productName) {
+			log.Printf("%s not specified, skipping generation", productName)
+			return
+		}
+
+		log.Printf("%s: Generating files", productName)
+		providerToGenerate.Generate(*outputPath, productName, generateCode, generateDocs)
 	}
 }
diff --git a/mmv1/provider/terraform.go b/mmv1/provider/terraform.go
@@ -104,7 +104,7 @@ func (t *Terraform) GenerateObject(object api.Resource, outputFolder, productPat
 		t.GenerateResource(object, *templateData, outputFolder, generateCode, generateDocs)
 
 		if generateCode {
-			log.Printf("Generating %s tests", object.Name)
+			// log.Printf("Generating %s tests", object.Name)
 			t.GenerateResourceTests(object, *templateData, outputFolder)
 			t.GenerateResourceSweeper(object, *templateData, outputFolder)
 		}