Refactor generator to use the metadata generated by autorest instead …

…of analyzing output folders via `git diff` (Azure#13830)

* introduced metadata processor to replace the previous get packages
function

* refactor

* some comment to satisfy the linter

* adding more comment to satisfy the linter

* refine and fix linter

* add some comment to satisfy linter

* gofmt

* rename function to satisfy linter

generate_options.json

@@ -1,6 +1,6 @@
 {
   "autorestArguments": [
-    "[email protected]/autorest.go@~2.1.161",
+    "[email protected]/autorest.go@~2.1.162",
     "--go",
     "--verbose",
     "--go-sdk-folder=.",

tools/generator/autorest/autorest.go

@@ -1,136 +1,128 @@
 package autorest
 
 import (
-	"bufio"
-	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
-	"log"
-	"os"
 	"os/exec"
-	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model"
 )
 
-// Task describes a generation task
-type Task struct {
-	// AbsReadmeMd absolute path of the readme.md file to generate
-	AbsReadmeMd string
+// Generator collects all the related context of an autorest generation
+type Generator struct {
+	arguments []string
+	cmd       *exec.Cmd
 }
 
-// Execute executes the autorest task, and then invoke the after scripts
-// the error returned will be TaskError
-func (t *Task) Execute(options Options) error {
-	if err := t.executeAutorest(options.AutorestArguments); err != nil {
-		return err
+// NewGeneratorFromOptions returns a new Generator with the given model.Options
+func NewGeneratorFromOptions(o model.Options) *Generator {
+	return &Generator{
+		arguments: o.Arguments(),
 	}
+}
 
-	if err := t.executeAfterScript(options.AfterScripts); err != nil {
-		return err
-	}
+// WithOption appends an model.Option to the argument list of the autorest generation
+func (g *Generator) WithOption(option model.Option) *Generator {
+	g.arguments = append(g.arguments, option.Format())
+	return g
+}
 
-	return nil
+// WithTag appends a tag option to the autorest argument list
+func (g *Generator) WithTag(tag string) *Generator {
+	return g.WithOption(model.NewKeyValueOption("tag", tag))
+}
+
+// WithMultiAPI appends a multiapi flag to the autorest argument list
+func (g *Generator) WithMultiAPI() *Generator {
+	return g.WithOption(model.NewFlagOption("multiapi"))
 }
 
-func (t *Task) executeAutorest(options []string) error {
-	arguments := append(options, t.AbsReadmeMd)
-	c := exec.Command("autorest", arguments...)
-	log.Printf("Executing autorest with parameters: %+v", arguments)
+// WithMetadataOutput appends a `metadata-output-folder` option to the autorest argument list
+func (g *Generator) WithMetadataOutput(output string) *Generator {
+	return g.WithOption(model.NewKeyValueOption("metadata-output-folder", output))
+}
 
-	stdout, _ := c.StdoutPipe()
-	stderr, _ := c.StderrPipe()
-	errScanner := bufio.NewScanner(stderr)
-	errScanner.Split(bufio.ScanLines)
-	outScanner := bufio.NewScanner(stdout)
-	outScanner.Split(bufio.ScanLines)
+// WithReadme appends a readme argument
+func (g *Generator) WithReadme(readme string) *Generator {
+	return g.WithOption(model.NewArgument(readme))
+}
 
-	if err := c.Start(); err != nil {
-		return &TaskError{
-			AbsReadmeMd: t.AbsReadmeMd,
-			Script:      "autorest",
-			Message:     err.Error(),
+// Generate executes the autorest generation. The error will be of type *GenerateError
+func (g *Generator) Generate() error {
+	g.buildCommand()
+	c := exec.Command("autorest", g.arguments...)
+	o, err := c.CombinedOutput()
+	if err != nil {
+		return &GenerateError{
+			Arguments: g.arguments,
+			Message:   string(o),
 		}
 	}
+	return nil
+}
 
-	go func() {
-		for errScanner.Scan() {
-			line := errScanner.Text()
-			fmt.Fprintln(os.Stderr, "[AUTOREST] "+line)
-		}
-	}()
-
-	go func() {
-		for outScanner.Scan() {
-			line := outScanner.Text()
-			fmt.Fprintln(os.Stdout, "[AUTOREST] "+line)
-		}
-	}()
+func (g *Generator) buildCommand() {
+	if g.cmd != nil {
+		return
+	}
+	g.cmd = exec.Command("autorest", g.arguments...)
+}
 
-	if err := c.Wait(); err != nil {
-		return &TaskError{
-			AbsReadmeMd: t.AbsReadmeMd,
-			Script:      "autorest",
-			Message:     err.Error(),
+// Start starts the generation
+func (g *Generator) Start() error {
+	g.buildCommand()
+	if err := g.cmd.Start(); err != nil {
+		return &GenerateError{
+			Arguments: g.arguments,
+			Message:   err.Error(),
 		}
 	}
 	return nil
 }
 
-func (t *Task) executeAfterScript(afterScripts []string) error {
-	for _, script := range afterScripts {
-		log.Printf("Executing after scripts %s...", script)
-		arguments := strings.Split(script, " ")
-		c := exec.Command(arguments[0], arguments[1:]...)
-		output, err := c.CombinedOutput()
-		if err != nil {
-			return &TaskError{
-				AbsReadmeMd: t.AbsReadmeMd,
-				Script:      script,
-				Message:     string(output),
-			}
+// Wait waits for the generation to complete
+func (g *Generator) Wait() error {
+	g.buildCommand()
+	if err := g.cmd.Wait(); err != nil {
+		return &GenerateError{
+			Arguments: g.arguments,
+			Message:   err.Error(),
 		}
 	}
-
 	return nil
 }
 
-// Options describes the options used in an autorest task
-type Options struct {
-	// AutorestArguments are the optional flags for the autorest tool
-	AutorestArguments []string
-	// AfterScripts are the scripts that need to be run after the SDK is generated
-	AfterScripts []string
+// Run starts and waits the generation
+func (g *Generator) Run() error {
+	g.buildCommand()
+	if err := g.cmd.Run(); err != nil {
+		return &GenerateError{
+			Arguments: g.arguments,
+			Message:   err.Error(),
+		}
+	}
+	return nil
 }
 
-// NewOptionsFrom returns a new options from a io.Reader
-func NewOptionsFrom(reader io.Reader) (*Options, error) {
-	b, err := ioutil.ReadAll(reader)
-	if err != nil {
-		return nil, err
-	}
-	var result Options
-	if err := json.Unmarshal(b, &result); err != nil {
-		return nil, err
-	}
-	return &result, nil
+// StdoutPipe returns the stdout pipeline of the command
+func (g *Generator) StdoutPipe() (io.ReadCloser, error) {
+	g.buildCommand()
+	return g.cmd.StdoutPipe()
 }
 
-// String ...
-func (o Options) String() string {
-	b, _ := json.MarshalIndent(o, "", "  ")
-	return string(b)
+// StderrPipe returns the stderr pipeline of the command
+func (g *Generator) StderrPipe() (io.ReadCloser, error) {
+	g.buildCommand()
+	return g.cmd.StderrPipe()
 }
 
-// TaskError the error returned during an autorest task
-type TaskError struct {
-	// AbsReadmeMd relative path of the readme.md file to generate
-	AbsReadmeMd string
-	// Script the script running when the error is thrown
-	Script string
-	// Message the error message
-	Message string
+// GenerateError ...
+type GenerateError struct {
+	Arguments []string
+	Message   string
 }
 
-func (r *TaskError) Error() string {
-	return fmt.Sprintf("autorest task failed for readme file '%s' during '%s': %s", r.AbsReadmeMd, r.Script, r.Message)
+// Error ...
+func (e *GenerateError) Error() string {
+	return fmt.Sprintf("autorest error with arguments '%s': \n%s", e.Arguments, e.Message)
 }

tools/generator/autorest/format.go

@@ -0,0 +1,16 @@
+package autorest
+
+import (
+	"fmt"
+	"os/exec"
+)
+
+// FormatPackage formats the given package using gofmt
+func FormatPackage(dir string) error {
+	c := exec.Command("gofmt", "-w", "-s", dir)
+	b, err := c.CombinedOutput()
+	if err != nil {
+		return fmt.Errorf(string(b))
+	}
+	return nil
+}

tools/generator/autorest/metadata.go

@@ -0,0 +1,77 @@
+package autorest
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/Azure/azure-sdk-for-go/tools/generator/autorest/model"
+)
+
+// MetadataProcessError ...
+type MetadataProcessError struct {
+	MetadataLocation string
+	Errors           []error
+}
+
+// Error ...
+func (e *MetadataProcessError) Error() string {
+	return fmt.Sprintf("total %d error(s) during processing metadata %s: %+v", len(e.Errors), e.MetadataLocation, e.Errors)
+}
+
+func (e *MetadataProcessError) add(err error) {
+	e.Errors = append(e.Errors, err)
+}
+
+// MetadataProcessor processes the metadata
+type MetadataProcessor struct {
+	metadataOutputFolder string
+}
+
+// NewMetadataProcessorFromLocation creates a new MetadataProcessor using the metadata output folder location
+func NewMetadataProcessorFromLocation(metadataOutput string) *MetadataProcessor {
+	return &MetadataProcessor{
+		metadataOutputFolder: metadataOutput,
+	}
+}
+
+// Process returns the metadata result: a map from tag to Metadata, and an error if there is anything that could not be processed.
+// the error returned must be of type *MetadataProcessError
+func (p MetadataProcessor) Process() (map[string]model.Metadata, error) {
+	fi, err := ioutil.ReadDir(p.metadataOutputFolder)
+	if err != nil {
+		return nil, &MetadataProcessError{
+			MetadataLocation: p.metadataOutputFolder,
+			Errors:           []error{err},
+		}
+	}
+	result := make(map[string]model.Metadata)
+	metadataErr := &MetadataProcessError{
+		MetadataLocation: p.metadataOutputFolder,
+	}
+	for _, f := range fi {
+		// a metadata output must be a json file
+		if f.IsDir() || !strings.HasSuffix(f.Name(), ".json") {
+			continue
+		}
+		file, err := os.Open(filepath.Join(p.metadataOutputFolder, f.Name()))
+		if err != nil {
+			metadataErr.add(err)
+			continue
+		}
+		metadata, err := model.NewMetadataFrom(file)
+		if err != nil {
+			metadataErr.add(err)
+			continue
+		}
+		tag := strings.TrimSuffix(f.Name(), ".json")
+		result[tag] = metadata
+	}
+
+	if len(metadataErr.Errors) != 0 {
+		return result, metadataErr
+	}
+	return result, nil
+}

tools/generator/autorest/model/metadata.go

@@ -0,0 +1,51 @@
+package model
+
+import (
+	"encoding/json"
+	"io"
+	"io/ioutil"
+)
+
+// Metadata ...
+type Metadata interface {
+	// SwaggerFiles returns the related swagger files in this tag
+	SwaggerFiles() []string
+	// PackagePath returns the output package path of this tag
+	PackagePath() string
+}
+
+// NewMetadataFrom reads a new Metadata from a io.Reader
+func NewMetadataFrom(reader io.Reader) (Metadata, error) {
+	b, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+	var result localMetadata
+	if err := json.Unmarshal(b, &result); err != nil {
+		return nil, err
+	}
+	return &result, nil
+}
+
+type localMetadata struct {
+	// InputFiles ...
+	InputFiles []string `json:"inputFiles"`
+	// OutputFolder ...
+	OutputFolder string `json:"outputFolder"`
+}
+
+// SwaggerFiles ...
+func (m localMetadata) SwaggerFiles() []string {
+	return m.InputFiles
+}
+
+// PackagePath ...
+func (m localMetadata) PackagePath() string {
+	return m.OutputFolder
+}
+
+// String ...
+func (m localMetadata) String() string {
+	b, _ := json.MarshalIndent(m, "", "  ")
+	return string(b)
+}