Parallelize aztft and azlist

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/hashicorp/terraform-exec v0.17.2
 	github.com/hexops/gotextdiff v1.0.3
 	github.com/magodo/armid v0.0.0-20220923023118-aec41eaf7370
-	github.com/magodo/azlist v0.0.0-20221201100009-664a9dd10ef3
+	github.com/magodo/azlist v0.0.0-20230118083100-39d50ebb1596
 	github.com/magodo/aztft v0.3.1-0.20230106111449-dcd315087da2
 	github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6
 	github.com/magodo/textinput v0.0.0-20210913072708-7d24f2b4b0c0

go.sum

@@ -192,8 +192,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/magodo/armid v0.0.0-20220923023118-aec41eaf7370 h1:n8RrB7jcZ9lQE7tyF2a7AEJ3Ux/E6E6FTeZLfgadPmg=
 github.com/magodo/armid v0.0.0-20220923023118-aec41eaf7370/go.mod h1:rR8E7zfGMbmfnSQvrkFiWYdhrfTqsVSltelnZB09BwA=
-github.com/magodo/azlist v0.0.0-20221201100009-664a9dd10ef3 h1:edFobje+hvzs8Vx2Lb8qRZLGN2OOtWoXuUQGH9BdAec=
-github.com/magodo/azlist v0.0.0-20221201100009-664a9dd10ef3/go.mod h1:r1a269lM5tSby3J7PaEUZQCKWgCLKz9KSWgs7u6fR/M=
+github.com/magodo/azlist v0.0.0-20230118083100-39d50ebb1596 h1:LsiNgiXcXFU7vJQRR+YeZCk0W4rVo5QfXxcg2wLxf3o=
+github.com/magodo/azlist v0.0.0-20230118083100-39d50ebb1596/go.mod h1:r1a269lM5tSby3J7PaEUZQCKWgCLKz9KSWgs7u6fR/M=
 github.com/magodo/aztft v0.3.1-0.20230106111449-dcd315087da2 h1:WxZPV+s4O53tQ0rSlBhzpP5lDwzmCDocg+TuICOK6XA=
 github.com/magodo/aztft v0.3.1-0.20230106111449-dcd315087da2/go.mod h1:UfUEt4CnhydYUqLTu7kmmx7CfFmQC9Y6cbyC87YA5VQ=
 github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6 h1:CElHO4hPXC+Eivy8sUC/WrnH3jmQzdF2x0lEXBEYul8=

internal/meta/meta_map.go

@@ -4,11 +4,12 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/Azure/aztfy/pkg/config"
-	"github.com/Azure/aztfy/pkg/log"
 	"os"
 	"sort"
 
+	"github.com/Azure/aztfy/pkg/config"
+	"github.com/Azure/aztfy/pkg/log"
+
 	"github.com/Azure/aztfy/internal/resmap"
 	"github.com/Azure/aztfy/internal/tfaddr"
 	"github.com/magodo/armid"
@@ -41,6 +42,7 @@ func (meta MetaMap) ScopeName() string {
 func (meta *MetaMap) ListResource(_ context.Context) (ImportList, error) {
 	var m resmap.ResourceMapping
 
+	log.Printf("[DEBUG] Read resource set from mapping file")
 	b, err := os.ReadFile(meta.mappingFile)
 	if err != nil {
 		return nil, fmt.Errorf("reading mapping file %s: %v", meta.mappingFile, err)

internal/meta/meta_query.go

@@ -3,6 +3,7 @@ package meta
 import (
 	"context"
 	"fmt"
+
 	"github.com/Azure/aztfy/internal/resourceset"
 	"github.com/Azure/aztfy/internal/tfaddr"
 	"github.com/Azure/aztfy/pkg/config"
@@ -44,18 +45,22 @@ func (meta MetaQuery) ScopeName() string {
 }
 
 func (meta *MetaQuery) ListResource(ctx context.Context) (ImportList, error) {
+	log.Printf("[DEBUG] Query resource set")
 	rset, err := meta.queryResourceSet(ctx, meta.argPredicate, meta.recursiveQuery)
 	if err != nil {
 		return nil, err
 	}
+	log.Printf("[DEBUG] Populate resource set")
 	if err := rset.PopulateResource(); err != nil {
 		return nil, fmt.Errorf("tweaking single resources in the azure resource set: %v", err)
 	}
+	log.Printf("[DEBUG] Reduce resource set")
 	if err := rset.ReduceResource(); err != nil {
 		return nil, fmt.Errorf("tweaking across resources in the azure resource set: %v", err)
 	}
 
-	rl := rset.ToTFResources()
+	log.Printf("[DEBUG] Azure Resource set map to TF resource set")
+	rl := rset.ToTFResources(meta.parallelism)
 
 	var l ImportList
 	for i, res := range rl {

internal/meta/meta_res.go

@@ -3,6 +3,7 @@ package meta
 import (
 	"context"
 	"fmt"
+
 	"github.com/Azure/aztfy/internal/resourceset"
 	"github.com/Azure/aztfy/internal/tfaddr"
 	"github.com/Azure/aztfy/pkg/config"
@@ -50,7 +51,8 @@ func (meta *MetaResource) ListResource(_ context.Context) (ImportList, error) {
 			},
 		},
 	}
-	rl := resourceSet.ToTFResources()
+	log.Printf("[DEBUG] Azure Resource set map to TF resource set")
+	rl := resourceSet.ToTFResources(meta.parallelism)
 
 	// This is to record known resource types. In case there is a known resource type and there comes another same typed resource,
 	// then we need to modify the resource name. Otherwise, there will be a resource address conflict.

internal/meta/meta_rg.go

@@ -3,6 +3,7 @@ package meta
 import (
 	"context"
 	"fmt"
+
 	"github.com/Azure/aztfy/internal/resourceset"
 	"github.com/Azure/aztfy/internal/tfaddr"
 	"github.com/Azure/aztfy/pkg/config"
@@ -39,18 +40,22 @@ func (meta MetaResourceGroup) ScopeName() string {
 }
 
 func (meta *MetaResourceGroup) ListResource(ctx context.Context) (ImportList, error) {
+	log.Printf("[DEBUG] Query resource set")
 	rset, err := meta.queryResourceSet(ctx, meta.resourceGroup)
 	if err != nil {
 		return nil, err
 	}
+	log.Printf("[DEBUG] Populate resource set")
 	if err := rset.PopulateResource(); err != nil {
 		return nil, fmt.Errorf("tweaking single resources in the azure resource set: %v", err)
 	}
+	log.Printf("[DEBUG] Reduce resource set")
 	if err := rset.ReduceResource(); err != nil {
 		return nil, fmt.Errorf("tweaking across resources in the azure resource set: %v", err)
 	}
 
-	rl := rset.ToTFResources()
+	log.Printf("[DEBUG] Azure Resource set map to TF resource set")
+	rl := rset.ToTFResources(meta.parallelism)
 
 	var l ImportList
 	for i, res := range rl {

internal/resourceset/azure_resource_set.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/magodo/armid"
 	"github.com/magodo/aztft/aztft"
+	"github.com/magodo/workerpool"
 )
 
 type AzureResourceSet struct {
@@ -23,42 +24,69 @@ type PesudoResourceInfo struct {
 	TFId   string
 }
 
-func (rset AzureResourceSet) ToTFResources() []TFResource {
+func (rset AzureResourceSet) ToTFResources(parallelism int) []TFResource {
 	tfresources := []TFResource{}
-	for _, res := range rset.Resources {
-		azureId := res.Id.String()
-		tftypes, tfids, exact, err := aztft.QueryTypeAndId(azureId, true)
-		if err != nil {
-			log.Printf("[WARN] Failed to query resource type for %s: %v\n", azureId, err)
+
+	wp := workerpool.NewWorkPool(parallelism)
+
+	type result struct {
+		resid   armid.ResourceId
+		tftypes []aztft.Type
+		tfids   []string
+		exact   bool
+		err     error
+	}
+
+	wp.Run(func(v interface{}) error {
+		res := v.(result)
+		if res.err != nil {
+			log.Printf("[WARN] Failed to query resource type for %s: %v\n", res.resid, res.err)
 			// Still put this unresolved resource in the resource set, so that users can later specify the expected TF resource type.
 			tfresources = append(tfresources, TFResource{
-				AzureId: res.Id,
+				AzureId: res.resid,
 				// Use the azure ID as the TF ID as a fallback
-				TFId: azureId,
+				TFId: res.resid.String(),
 			})
 		} else {
-			if !exact {
+			if !res.exact {
 				// It is not possible to return multiple result when API is used.
-				log.Printf("[WARN] No query result for resource type and TF id for %s\n", azureId)
+				log.Printf("[WARN] No query result for resource type and TF id for %s\n", res.resid)
 				// Still put this unresolved resource in the resource set, so that users can later specify the expected TF resource type.
 				tfresources = append(tfresources, TFResource{
-					AzureId: res.Id,
+					AzureId: res.resid,
 					// Use the azure ID as the TF ID as a fallback
-					TFId: azureId,
+					TFId: res.resid.String(),
 				})
 			} else {
-				for i := range tfids {
+				for i := range res.tfids {
 					tfresources = append(tfresources, TFResource{
-						AzureId: tftypes[i].AzureId,
-						TFId:    tfids[i],
-						TFType:  tftypes[i].TFType,
+						AzureId: res.tftypes[i].AzureId,
+						TFId:    res.tfids[i],
+						TFType:  res.tftypes[i].TFType,
 					})
 				}
 			}
 		}
+		return nil
+	})
 
+	for _, res := range rset.Resources {
+		res := res
+		wp.AddTask(func() (interface{}, error) {
+			tftypes, tfids, exact, err := aztft.QueryTypeAndId(res.Id.String(), true)
+			return result{
+				resid:   res.Id,
+				tftypes: tftypes,
+				tfids:   tfids,
+				exact:   exact,
+				err:     err,
+			}, nil
+		})
 	}
 
+	// #nosec G104
+	wp.Done()
+
 	sort.Slice(tfresources, func(i, j int) bool {
 		return tfresources[i].AzureId.String() < tfresources[j].AzureId.String()
 	})

internal/run.go

@@ -3,10 +3,11 @@ package internal
 import (
 	"context"
 	"fmt"
-	internalmeta "github.com/Azure/aztfy/internal/meta"
 	"os"
 	"strings"
 
+	internalmeta "github.com/Azure/aztfy/internal/meta"
+
 	"github.com/Azure/aztfy/internal/config"
 	"github.com/Azure/aztfy/pkg/meta"
 
@@ -118,7 +119,7 @@ func BatchImport(ctx context.Context, cfg config.NonInteractiveModeConfig) error
 
 	var err error
 	if cfg.PlainUI {
-		err = f(&StdoutMessager{})
+		err = f(NewStdoutMessager())
 	} else {
 		s := bspinner.NewModel()
 		s.Spinner = common.Spinner

internal/ui.go

@@ -1,19 +1,30 @@
 package internal
 
-import "fmt"
+import (
+	"log"
+	"os"
+)
 
 // Abstract the Messager struct in the github.com/magodo/spinner
 type Messager interface {
 	SetStatus(msg string)
 	SetDetail(msg string)
 }
 
-type StdoutMessager struct{}
+type stdoutMessager struct {
+	*log.Logger
+}
+
+func NewStdoutMessager() Messager {
+	return &stdoutMessager{
+		Logger: log.New(os.Stdout, "[aztfy] ", log.LstdFlags),
+	}
+}
 
-func (p *StdoutMessager) SetStatus(msg string) {
-	fmt.Println(msg)
+func (p *stdoutMessager) SetStatus(msg string) {
+	p.Println(msg)
 }
 
-func (p *StdoutMessager) SetDetail(msg string) {
-	fmt.Println(msg)
+func (p *stdoutMessager) SetDetail(msg string) {
+	p.Println(msg)
 }