diff --git a/go.mod b/go.mod index 8d180a5..4ea6e93 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/hashicorp/terraform-exec v0.17.2 github.com/hexops/gotextdiff v1.0.3 github.com/magodo/armid v0.0.0-20220923023118-aec41eaf7370 - github.com/magodo/azlist v0.0.0-20221201100009-664a9dd10ef3 + github.com/magodo/azlist v0.0.0-20230118083100-39d50ebb1596 github.com/magodo/aztft v0.3.1-0.20230106111449-dcd315087da2 github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6 github.com/magodo/textinput v0.0.0-20210913072708-7d24f2b4b0c0 diff --git a/go.sum b/go.sum index b2baea7..384ac4b 100644 --- a/go.sum +++ b/go.sum @@ -192,8 +192,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/magodo/armid v0.0.0-20220923023118-aec41eaf7370 h1:n8RrB7jcZ9lQE7tyF2a7AEJ3Ux/E6E6FTeZLfgadPmg= github.com/magodo/armid v0.0.0-20220923023118-aec41eaf7370/go.mod h1:rR8E7zfGMbmfnSQvrkFiWYdhrfTqsVSltelnZB09BwA= -github.com/magodo/azlist v0.0.0-20221201100009-664a9dd10ef3 h1:edFobje+hvzs8Vx2Lb8qRZLGN2OOtWoXuUQGH9BdAec= -github.com/magodo/azlist v0.0.0-20221201100009-664a9dd10ef3/go.mod h1:r1a269lM5tSby3J7PaEUZQCKWgCLKz9KSWgs7u6fR/M= +github.com/magodo/azlist v0.0.0-20230118083100-39d50ebb1596 h1:LsiNgiXcXFU7vJQRR+YeZCk0W4rVo5QfXxcg2wLxf3o= +github.com/magodo/azlist v0.0.0-20230118083100-39d50ebb1596/go.mod h1:r1a269lM5tSby3J7PaEUZQCKWgCLKz9KSWgs7u6fR/M= github.com/magodo/aztft v0.3.1-0.20230106111449-dcd315087da2 h1:WxZPV+s4O53tQ0rSlBhzpP5lDwzmCDocg+TuICOK6XA= github.com/magodo/aztft v0.3.1-0.20230106111449-dcd315087da2/go.mod h1:UfUEt4CnhydYUqLTu7kmmx7CfFmQC9Y6cbyC87YA5VQ= github.com/magodo/spinner v0.0.0-20220720073946-50f31b2dc5a6 h1:CElHO4hPXC+Eivy8sUC/WrnH3jmQzdF2x0lEXBEYul8= diff --git a/internal/meta/meta_map.go b/internal/meta/meta_map.go index da6bd1c..6efbd8b 100644 --- a/internal/meta/meta_map.go +++ b/internal/meta/meta_map.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "fmt" - "github.com/Azure/aztfy/pkg/config" - "github.com/Azure/aztfy/pkg/log" "os" "sort" + "github.com/Azure/aztfy/pkg/config" + "github.com/Azure/aztfy/pkg/log" + "github.com/Azure/aztfy/internal/resmap" "github.com/Azure/aztfy/internal/tfaddr" "github.com/magodo/armid" @@ -41,6 +42,7 @@ func (meta MetaMap) ScopeName() string { func (meta *MetaMap) ListResource(_ context.Context) (ImportList, error) { var m resmap.ResourceMapping + log.Printf("[DEBUG] Read resource set from mapping file") b, err := os.ReadFile(meta.mappingFile) if err != nil { return nil, fmt.Errorf("reading mapping file %s: %v", meta.mappingFile, err) diff --git a/internal/meta/meta_query.go b/internal/meta/meta_query.go index d62f364..e475a6a 100644 --- a/internal/meta/meta_query.go +++ b/internal/meta/meta_query.go @@ -3,6 +3,7 @@ package meta import ( "context" "fmt" + "github.com/Azure/aztfy/internal/resourceset" "github.com/Azure/aztfy/internal/tfaddr" "github.com/Azure/aztfy/pkg/config" @@ -44,18 +45,22 @@ func (meta MetaQuery) ScopeName() string { } func (meta *MetaQuery) ListResource(ctx context.Context) (ImportList, error) { + log.Printf("[DEBUG] Query resource set") rset, err := meta.queryResourceSet(ctx, meta.argPredicate, meta.recursiveQuery) if err != nil { return nil, err } + log.Printf("[DEBUG] Populate resource set") if err := rset.PopulateResource(); err != nil { return nil, fmt.Errorf("tweaking single resources in the azure resource set: %v", err) } + log.Printf("[DEBUG] Reduce resource set") if err := rset.ReduceResource(); err != nil { return nil, fmt.Errorf("tweaking across resources in the azure resource set: %v", err) } - rl := rset.ToTFResources() + log.Printf("[DEBUG] Azure Resource set map to TF resource set") + rl := rset.ToTFResources(meta.parallelism) var l ImportList for i, res := range rl { diff --git a/internal/meta/meta_res.go b/internal/meta/meta_res.go index 5c73e50..95ae2a3 100644 --- a/internal/meta/meta_res.go +++ b/internal/meta/meta_res.go @@ -3,6 +3,7 @@ package meta import ( "context" "fmt" + "github.com/Azure/aztfy/internal/resourceset" "github.com/Azure/aztfy/internal/tfaddr" "github.com/Azure/aztfy/pkg/config" @@ -50,7 +51,8 @@ func (meta *MetaResource) ListResource(_ context.Context) (ImportList, error) { }, }, } - rl := resourceSet.ToTFResources() + log.Printf("[DEBUG] Azure Resource set map to TF resource set") + rl := resourceSet.ToTFResources(meta.parallelism) // This is to record known resource types. In case there is a known resource type and there comes another same typed resource, // then we need to modify the resource name. Otherwise, there will be a resource address conflict. diff --git a/internal/meta/meta_rg.go b/internal/meta/meta_rg.go index 11b937f..5fb7f44 100644 --- a/internal/meta/meta_rg.go +++ b/internal/meta/meta_rg.go @@ -3,6 +3,7 @@ package meta import ( "context" "fmt" + "github.com/Azure/aztfy/internal/resourceset" "github.com/Azure/aztfy/internal/tfaddr" "github.com/Azure/aztfy/pkg/config" @@ -39,18 +40,22 @@ func (meta MetaResourceGroup) ScopeName() string { } func (meta *MetaResourceGroup) ListResource(ctx context.Context) (ImportList, error) { + log.Printf("[DEBUG] Query resource set") rset, err := meta.queryResourceSet(ctx, meta.resourceGroup) if err != nil { return nil, err } + log.Printf("[DEBUG] Populate resource set") if err := rset.PopulateResource(); err != nil { return nil, fmt.Errorf("tweaking single resources in the azure resource set: %v", err) } + log.Printf("[DEBUG] Reduce resource set") if err := rset.ReduceResource(); err != nil { return nil, fmt.Errorf("tweaking across resources in the azure resource set: %v", err) } - rl := rset.ToTFResources() + log.Printf("[DEBUG] Azure Resource set map to TF resource set") + rl := rset.ToTFResources(meta.parallelism) var l ImportList for i, res := range rl { diff --git a/internal/resourceset/azure_resource_set.go b/internal/resourceset/azure_resource_set.go index 39ab17b..6f0ba24 100644 --- a/internal/resourceset/azure_resource_set.go +++ b/internal/resourceset/azure_resource_set.go @@ -7,6 +7,7 @@ import ( "github.com/magodo/armid" "github.com/magodo/aztft/aztft" + "github.com/magodo/workerpool" ) type AzureResourceSet struct { @@ -23,42 +24,69 @@ type PesudoResourceInfo struct { TFId string } -func (rset AzureResourceSet) ToTFResources() []TFResource { +func (rset AzureResourceSet) ToTFResources(parallelism int) []TFResource { tfresources := []TFResource{} - for _, res := range rset.Resources { - azureId := res.Id.String() - tftypes, tfids, exact, err := aztft.QueryTypeAndId(azureId, true) - if err != nil { - log.Printf("[WARN] Failed to query resource type for %s: %v\n", azureId, err) + + wp := workerpool.NewWorkPool(parallelism) + + type result struct { + resid armid.ResourceId + tftypes []aztft.Type + tfids []string + exact bool + err error + } + + wp.Run(func(v interface{}) error { + res := v.(result) + if res.err != nil { + log.Printf("[WARN] Failed to query resource type for %s: %v\n", res.resid, res.err) // Still put this unresolved resource in the resource set, so that users can later specify the expected TF resource type. tfresources = append(tfresources, TFResource{ - AzureId: res.Id, + AzureId: res.resid, // Use the azure ID as the TF ID as a fallback - TFId: azureId, + TFId: res.resid.String(), }) } else { - if !exact { + if !res.exact { // It is not possible to return multiple result when API is used. - log.Printf("[WARN] No query result for resource type and TF id for %s\n", azureId) + log.Printf("[WARN] No query result for resource type and TF id for %s\n", res.resid) // Still put this unresolved resource in the resource set, so that users can later specify the expected TF resource type. tfresources = append(tfresources, TFResource{ - AzureId: res.Id, + AzureId: res.resid, // Use the azure ID as the TF ID as a fallback - TFId: azureId, + TFId: res.resid.String(), }) } else { - for i := range tfids { + for i := range res.tfids { tfresources = append(tfresources, TFResource{ - AzureId: tftypes[i].AzureId, - TFId: tfids[i], - TFType: tftypes[i].TFType, + AzureId: res.tftypes[i].AzureId, + TFId: res.tfids[i], + TFType: res.tftypes[i].TFType, }) } } } + return nil + }) + for _, res := range rset.Resources { + res := res + wp.AddTask(func() (interface{}, error) { + tftypes, tfids, exact, err := aztft.QueryTypeAndId(res.Id.String(), true) + return result{ + resid: res.Id, + tftypes: tftypes, + tfids: tfids, + exact: exact, + err: err, + }, nil + }) } + // #nosec G104 + wp.Done() + sort.Slice(tfresources, func(i, j int) bool { return tfresources[i].AzureId.String() < tfresources[j].AzureId.String() }) diff --git a/internal/run.go b/internal/run.go index 18536d0..f18c1ed 100644 --- a/internal/run.go +++ b/internal/run.go @@ -3,10 +3,11 @@ package internal import ( "context" "fmt" - internalmeta "github.com/Azure/aztfy/internal/meta" "os" "strings" + internalmeta "github.com/Azure/aztfy/internal/meta" + "github.com/Azure/aztfy/internal/config" "github.com/Azure/aztfy/pkg/meta" @@ -118,7 +119,7 @@ func BatchImport(ctx context.Context, cfg config.NonInteractiveModeConfig) error var err error if cfg.PlainUI { - err = f(&StdoutMessager{}) + err = f(NewStdoutMessager()) } else { s := bspinner.NewModel() s.Spinner = common.Spinner diff --git a/internal/ui.go b/internal/ui.go index 1db46c6..18cd001 100644 --- a/internal/ui.go +++ b/internal/ui.go @@ -1,6 +1,9 @@ package internal -import "fmt" +import ( + "log" + "os" +) // Abstract the Messager struct in the github.com/magodo/spinner type Messager interface { @@ -8,12 +11,20 @@ type Messager interface { SetDetail(msg string) } -type StdoutMessager struct{} +type stdoutMessager struct { + *log.Logger +} + +func NewStdoutMessager() Messager { + return &stdoutMessager{ + Logger: log.New(os.Stdout, "[aztfy] ", log.LstdFlags), + } +} -func (p *StdoutMessager) SetStatus(msg string) { - fmt.Println(msg) +func (p *stdoutMessager) SetStatus(msg string) { + p.Println(msg) } -func (p *StdoutMessager) SetDetail(msg string) { - fmt.Println(msg) +func (p *stdoutMessager) SetDetail(msg string) { + p.Println(msg) }