From 86143a6389aa330e3ab11ab41664d07424c676e0 Mon Sep 17 00:00:00 2001 From: Jared Palmer Date: Thu, 16 Dec 2021 16:31:43 -0500 Subject: [PATCH] Fix doublestar workspace bug (#310) --- cli/go.mod | 2 +- cli/go.sum | 4 +- cli/internal/context/context.go | 48 +++++++++---------- cli/internal/fs/fs.go | 25 ---------- cli/internal/fs/globby/globby.go | 79 ++++++++++++++++++++++++++++++++ cli/internal/run/run.go | 20 ++------ cli/scripts/monorepo.ts | 2 +- 7 files changed, 111 insertions(+), 69 deletions(-) create mode 100644 cli/internal/fs/globby/globby.go diff --git a/cli/go.mod b/cli/go.mod index dd0b146eebd47..34ff75c130d28 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -6,7 +6,7 @@ require ( github.com/AlecAivazis/survey/v2 v2.2.12 github.com/adrg/xdg v0.3.3 github.com/armon/go-radix v1.0.0 // indirect - github.com/bmatcuk/doublestar v1.3.4 + github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/briandowns/spinner v1.16.0 github.com/deckarep/golang-set v1.7.1 github.com/fatih/color v1.7.0 diff --git a/cli/go.sum b/cli/go.sum index e474a9957265d..7d793cfd72e47 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -15,8 +15,8 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= -github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= +github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/briandowns/spinner v1.16.0 h1:DFmp6hEaIx2QXXuqSJmtfSBSAjRmpGiKG6ip2Wm/yOs= github.com/briandowns/spinner v1.16.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index c2efd16bd4f0b..3cc307a6d7d01 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -2,7 +2,7 @@ package context import ( "fmt" - "log" + "path" "path/filepath" "sort" "strings" @@ -11,9 +11,9 @@ import ( "turbo/internal/backends" "turbo/internal/config" "turbo/internal/fs" + "turbo/internal/fs/globby" "turbo/internal/util" - "github.com/bmatcuk/doublestar" mapset "github.com/deckarep/golang-set" "github.com/fatih/color" "github.com/google/chrometracing" @@ -180,14 +180,9 @@ func WithGraph(rootpath string, config *config.Config) Option { globalDeps := make(util.Set) if len(pkg.Turbo.GlobalDependencies) > 0 { - for _, value := range pkg.Turbo.GlobalDependencies { - f, err := filepath.Glob(value) - if err != nil { - return fmt.Errorf("error parsing global dependencies glob %v: %w", value, err) - } - for _, val := range f { - globalDeps.Add(val) - } + f := globby.GlobFiles(rootpath, &pkg.Turbo.GlobalDependencies, nil) + for _, val := range f { + globalDeps.Add(val) } } if c.Backend.Name != "nodejs-yarn" || fs.CheckIfWindows() { @@ -237,18 +232,24 @@ func WithGraph(rootpath string, config *config.Config) Option { // until all parsing is complete // and populate the graph parseJSONWaitGroup := new(errgroup.Group) - for _, value := range spaces { - f, err := doublestar.Glob(value) - if err != nil { - log.Fatalf("Error parsing workspaces glob %v", value) - } + justJsons := make([]string, len(spaces)) + for i, space := range spaces { + justJsons[i] = path.Join(space, "package.json") + } + ignore := []string{ + "**/node_modules/**/*", + "**/bower_components/**/*", + "**/test/**/*", + "**/tests/**/*", + } - for i, val := range f { - _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines - parseJSONWaitGroup.Go(func() error { - return c.parsePackageJSON(val) - }) - } + f := globby.GlobFiles(rootpath, &justJsons, &ignore) + + for i, val := range f { + _, val := i, val // https://golang.org/doc/faq#closures_and_goroutines + parseJSONWaitGroup.Go(func() error { + return c.parsePackageJSON(val) + }) } if err := parseJSONWaitGroup.Wait(); err != nil { @@ -442,10 +443,9 @@ func (c *Context) populateTopologicGraphForPackageJson(pkg *fs.PackageJSON) erro return nil } -func (c *Context) parsePackageJSON(fileName string) error { +func (c *Context) parsePackageJSON(buildFilePath string) error { c.mutex.Lock() defer c.mutex.Unlock() - buildFilePath := filepath.Join(fileName, "package.json") // log.Printf("[TRACE] reading package.json : %+v", buildFilePath) if fs.FileExists(buildFilePath) { @@ -457,7 +457,7 @@ func (c *Context) parsePackageJSON(fileName string) error { // log.Printf("[TRACE] adding %+v to graph", pkg.Name) c.TopologicalGraph.Add(pkg.Name) pkg.PackageJSONPath = buildFilePath - pkg.Dir = fileName + pkg.Dir = filepath.Dir(buildFilePath) c.PackageInfos[pkg.Name] = pkg c.PackageNames = append(c.PackageNames, pkg.Name) } diff --git a/cli/internal/fs/fs.go b/cli/internal/fs/fs.go index 377a23d40765f..20e7970dd21ea 100644 --- a/cli/internal/fs/fs.go +++ b/cli/internal/fs/fs.go @@ -1,16 +1,11 @@ package fs import ( - "fmt" "io" "io/ioutil" "log" "os" "path/filepath" - "strings" - "turbo/internal/util" - - "github.com/bmatcuk/doublestar" ) // https://github.com/thought-machine/please/blob/master/src/fs/fs.go @@ -161,23 +156,3 @@ func copyFile(from, to string) (err error) { return nil } - -// GlobList accepts a list of doublestar directive globs and returns a list of files matching them -func Globby(globs []string) ([]string, error) { - var fileset = make(util.Set) - for _, output := range globs { - results, err := doublestar.Glob(strings.TrimPrefix(output, "!")) - if err != nil { - return nil, fmt.Errorf("invalid glob %v: %w", output, err) - } - // we handle negation via "!" by removing the result from the fileset - for _, result := range results { - if strings.HasPrefix(output, "!") { - fileset.Delete(result) - } else { - fileset.Add(result) - } - } - } - return fileset.UnsafeListOfStrings(), nil -} diff --git a/cli/internal/fs/globby/globby.go b/cli/internal/fs/globby/globby.go new file mode 100644 index 0000000000000..61c0d46c032cb --- /dev/null +++ b/cli/internal/fs/globby/globby.go @@ -0,0 +1,79 @@ +package globby + +import ( + "fmt" + "io/fs" + "path/filepath" + "strings" + + "github.com/bmatcuk/doublestar/v4" +) + +// // GlobList accepts a list of doublestar directive globs and returns a list of files matching them +// func Globby(base string, globs []string) ([]string, error) { +// ignoreList := []string{} +// actualGlobs := []string{} +// for _, output := range globs { +// if strings.HasPrefix(output, "!") { +// ignoreList = append(ignoreList, strings.TrimPrefix(output, "!")) +// } else { +// actualGlobs = append(actualGlobs, output) +// } +// } +// files := []string{} +// for _, glob := range actualGlobs { +// matches, err := doublestar.Glob(os.DirFS(base), glob) +// if err != nil { +// return nil, err +// } +// for _, match := range matches { +// for _, ignore := range ignoreList { +// if isMatch, _ := doublestar.PathMatch(ignore, match); !isMatch { +// files = append(files, match) +// } +// } +// } +// } +// } + +func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]string) []string { + var include []string + var exclude []string + var result []string + + for _, p := range *include_pattens { + include = append(include, filepath.Join(ws_path, p)) + } + + for _, p := range *exclude_pattens { + exclude = append(exclude, filepath.Join(ws_path, p)) + } + + var include_pattern = "{" + strings.Join(include, ",") + "}" + var exclude_pattern = "{" + strings.Join(exclude, ",") + "}" + var _ = filepath.Walk(ws_path, func(p string, info fs.FileInfo, err error) error { + if err != nil { + fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", p, err) + return err + } + + if val, _ := doublestar.PathMatch(exclude_pattern, p); val { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + if info.IsDir() { + return nil + } + + if val, _ := doublestar.PathMatch(include_pattern, p); val || len(*include_pattens) == 0 { + result = append(result, p) + } + + return nil + }) + + return result +} diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index 2439f7b4c963d..de66df041595c 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -20,11 +20,11 @@ import ( "turbo/internal/context" "turbo/internal/core" "turbo/internal/fs" + "turbo/internal/fs/globby" "turbo/internal/scm" "turbo/internal/ui" "turbo/internal/util" - "github.com/bmatcuk/doublestar" "github.com/pyr-sh/dag" "github.com/fatih/color" @@ -582,21 +582,9 @@ func (c *RunCommand) Run(args []string) int { if runOptions.cache && (pipeline.Cache == nil || *pipeline.Cache) { targetLogger.Debug("caching output", "outputs", outputs) - var filesToBeCached = make(util.Set) - for _, output := range outputs { - results, err := doublestar.Glob(filepath.Join(pack.Dir, strings.TrimPrefix(output, "!"))) - if err != nil { - targetUi.Error(fmt.Sprintf("Could not find output artifacts in %v, likely invalid glob %v: %s", pack.Dir, output, err)) - } - for _, result := range results { - if strings.HasPrefix(output, "!") { - filesToBeCached.Delete(result) - } else { - filesToBeCached.Add(result) - } - } - } - if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached.UnsafeListOfStrings()); err != nil { + ignore := []string{} + filesToBeCached := globby.GlobFiles(pack.Dir, &outputs, &ignore) + if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached); err != nil { c.logError(targetLogger, "", fmt.Errorf("Error caching output: %w", err)) } } diff --git a/cli/scripts/monorepo.ts b/cli/scripts/monorepo.ts index fa3f28780c970..7e3056655deea 100644 --- a/cli/scripts/monorepo.ts +++ b/cli/scripts/monorepo.ts @@ -141,7 +141,7 @@ importers: version: "0.1.0", private: true, license: "MIT", - workspaces: ["packages/*"], + workspaces: ["packages/**"], scripts: { build: `${turboPath} run build`, test: `${turboPath} run test`,