Skip to content

Commit

Permalink
Fix doublestar workspace bug (#310)
Browse files Browse the repository at this point in the history
  • Loading branch information
jaredpalmer authored Dec 16, 2021
1 parent 7dfdcd3 commit 86143a6
Show file tree
Hide file tree
Showing 7 changed files with 111 additions and 69 deletions.
2 changes: 1 addition & 1 deletion cli/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/AlecAivazis/survey/v2 v2.2.12
github.com/adrg/xdg v0.3.3
github.com/armon/go-radix v1.0.0 // indirect
github.com/bmatcuk/doublestar v1.3.4
github.com/bmatcuk/doublestar/v4 v4.0.2
github.com/briandowns/spinner v1.16.0
github.com/deckarep/golang-set v1.7.1
github.com/fatih/color v1.7.0
Expand Down
4 changes: 2 additions & 2 deletions cli/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/briandowns/spinner v1.16.0 h1:DFmp6hEaIx2QXXuqSJmtfSBSAjRmpGiKG6ip2Wm/yOs=
github.com/briandowns/spinner v1.16.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
Expand Down
48 changes: 24 additions & 24 deletions cli/internal/context/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package context

import (
"fmt"
"log"
"path"
"path/filepath"
"sort"
"strings"
Expand All @@ -11,9 +11,9 @@ import (
"turbo/internal/backends"
"turbo/internal/config"
"turbo/internal/fs"
"turbo/internal/fs/globby"
"turbo/internal/util"

"github.com/bmatcuk/doublestar"
mapset "github.com/deckarep/golang-set"
"github.com/fatih/color"
"github.com/google/chrometracing"
Expand Down Expand Up @@ -180,14 +180,9 @@ func WithGraph(rootpath string, config *config.Config) Option {
globalDeps := make(util.Set)

if len(pkg.Turbo.GlobalDependencies) > 0 {
for _, value := range pkg.Turbo.GlobalDependencies {
f, err := filepath.Glob(value)
if err != nil {
return fmt.Errorf("error parsing global dependencies glob %v: %w", value, err)
}
for _, val := range f {
globalDeps.Add(val)
}
f := globby.GlobFiles(rootpath, &pkg.Turbo.GlobalDependencies, nil)
for _, val := range f {
globalDeps.Add(val)
}
}
if c.Backend.Name != "nodejs-yarn" || fs.CheckIfWindows() {
Expand Down Expand Up @@ -237,18 +232,24 @@ func WithGraph(rootpath string, config *config.Config) Option {
// until all parsing is complete
// and populate the graph
parseJSONWaitGroup := new(errgroup.Group)
for _, value := range spaces {
f, err := doublestar.Glob(value)
if err != nil {
log.Fatalf("Error parsing workspaces glob %v", value)
}
justJsons := make([]string, len(spaces))
for i, space := range spaces {
justJsons[i] = path.Join(space, "package.json")
}
ignore := []string{
"**/node_modules/**/*",
"**/bower_components/**/*",
"**/test/**/*",
"**/tests/**/*",
}

for i, val := range f {
_, val := i, val // https://golang.org/doc/faq#closures_and_goroutines
parseJSONWaitGroup.Go(func() error {
return c.parsePackageJSON(val)
})
}
f := globby.GlobFiles(rootpath, &justJsons, &ignore)

for i, val := range f {
_, val := i, val // https://golang.org/doc/faq#closures_and_goroutines
parseJSONWaitGroup.Go(func() error {
return c.parsePackageJSON(val)
})
}

if err := parseJSONWaitGroup.Wait(); err != nil {
Expand Down Expand Up @@ -442,10 +443,9 @@ func (c *Context) populateTopologicGraphForPackageJson(pkg *fs.PackageJSON) erro
return nil
}

func (c *Context) parsePackageJSON(fileName string) error {
func (c *Context) parsePackageJSON(buildFilePath string) error {
c.mutex.Lock()
defer c.mutex.Unlock()
buildFilePath := filepath.Join(fileName, "package.json")

// log.Printf("[TRACE] reading package.json : %+v", buildFilePath)
if fs.FileExists(buildFilePath) {
Expand All @@ -457,7 +457,7 @@ func (c *Context) parsePackageJSON(fileName string) error {
// log.Printf("[TRACE] adding %+v to graph", pkg.Name)
c.TopologicalGraph.Add(pkg.Name)
pkg.PackageJSONPath = buildFilePath
pkg.Dir = fileName
pkg.Dir = filepath.Dir(buildFilePath)
c.PackageInfos[pkg.Name] = pkg
c.PackageNames = append(c.PackageNames, pkg.Name)
}
Expand Down
25 changes: 0 additions & 25 deletions cli/internal/fs/fs.go
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
package fs

import (
"fmt"
"io"
"io/ioutil"
"log"
"os"
"path/filepath"
"strings"
"turbo/internal/util"

"github.com/bmatcuk/doublestar"
)

// https://github.com/thought-machine/please/blob/master/src/fs/fs.go
Expand Down Expand Up @@ -161,23 +156,3 @@ func copyFile(from, to string) (err error) {

return nil
}

// GlobList accepts a list of doublestar directive globs and returns a list of files matching them
func Globby(globs []string) ([]string, error) {
var fileset = make(util.Set)
for _, output := range globs {
results, err := doublestar.Glob(strings.TrimPrefix(output, "!"))
if err != nil {
return nil, fmt.Errorf("invalid glob %v: %w", output, err)
}
// we handle negation via "!" by removing the result from the fileset
for _, result := range results {
if strings.HasPrefix(output, "!") {
fileset.Delete(result)
} else {
fileset.Add(result)
}
}
}
return fileset.UnsafeListOfStrings(), nil
}
79 changes: 79 additions & 0 deletions cli/internal/fs/globby/globby.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package globby

import (
"fmt"
"io/fs"
"path/filepath"
"strings"

"github.com/bmatcuk/doublestar/v4"
)

// // GlobList accepts a list of doublestar directive globs and returns a list of files matching them
// func Globby(base string, globs []string) ([]string, error) {
// ignoreList := []string{}
// actualGlobs := []string{}
// for _, output := range globs {
// if strings.HasPrefix(output, "!") {
// ignoreList = append(ignoreList, strings.TrimPrefix(output, "!"))
// } else {
// actualGlobs = append(actualGlobs, output)
// }
// }
// files := []string{}
// for _, glob := range actualGlobs {
// matches, err := doublestar.Glob(os.DirFS(base), glob)
// if err != nil {
// return nil, err
// }
// for _, match := range matches {
// for _, ignore := range ignoreList {
// if isMatch, _ := doublestar.PathMatch(ignore, match); !isMatch {
// files = append(files, match)
// }
// }
// }
// }
// }

func GlobFiles(ws_path string, include_pattens *[]string, exclude_pattens *[]string) []string {
var include []string
var exclude []string
var result []string

for _, p := range *include_pattens {
include = append(include, filepath.Join(ws_path, p))
}

for _, p := range *exclude_pattens {
exclude = append(exclude, filepath.Join(ws_path, p))
}

var include_pattern = "{" + strings.Join(include, ",") + "}"
var exclude_pattern = "{" + strings.Join(exclude, ",") + "}"
var _ = filepath.Walk(ws_path, func(p string, info fs.FileInfo, err error) error {
if err != nil {
fmt.Printf("prevent panic by handling failure accessing a path %q: %v\n", p, err)
return err
}

if val, _ := doublestar.PathMatch(exclude_pattern, p); val {
if info.IsDir() {
return filepath.SkipDir
}
return nil
}

if info.IsDir() {
return nil
}

if val, _ := doublestar.PathMatch(include_pattern, p); val || len(*include_pattens) == 0 {
result = append(result, p)
}

return nil
})

return result
}
20 changes: 4 additions & 16 deletions cli/internal/run/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@ import (
"turbo/internal/context"
"turbo/internal/core"
"turbo/internal/fs"
"turbo/internal/fs/globby"
"turbo/internal/scm"
"turbo/internal/ui"
"turbo/internal/util"

"github.com/bmatcuk/doublestar"
"github.com/pyr-sh/dag"

"github.com/fatih/color"
Expand Down Expand Up @@ -582,21 +582,9 @@ func (c *RunCommand) Run(args []string) int {

if runOptions.cache && (pipeline.Cache == nil || *pipeline.Cache) {
targetLogger.Debug("caching output", "outputs", outputs)
var filesToBeCached = make(util.Set)
for _, output := range outputs {
results, err := doublestar.Glob(filepath.Join(pack.Dir, strings.TrimPrefix(output, "!")))
if err != nil {
targetUi.Error(fmt.Sprintf("Could not find output artifacts in %v, likely invalid glob %v: %s", pack.Dir, output, err))
}
for _, result := range results {
if strings.HasPrefix(output, "!") {
filesToBeCached.Delete(result)
} else {
filesToBeCached.Add(result)
}
}
}
if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached.UnsafeListOfStrings()); err != nil {
ignore := []string{}
filesToBeCached := globby.GlobFiles(pack.Dir, &outputs, &ignore)
if err := turboCache.Put(pack.Dir, hash, int(time.Since(cmdTime).Milliseconds()), filesToBeCached); err != nil {
c.logError(targetLogger, "", fmt.Errorf("Error caching output: %w", err))
}
}
Expand Down
2 changes: 1 addition & 1 deletion cli/scripts/monorepo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ importers:
version: "0.1.0",
private: true,
license: "MIT",
workspaces: ["packages/*"],
workspaces: ["packages/**"],
scripts: {
build: `${turboPath} run build`,
test: `${turboPath} run test`,
Expand Down

1 comment on commit 86143a6

@vercel
Copy link

@vercel vercel bot commented on 86143a6 Dec 16, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.