Revert "Merge pull request moby#4094 from jedevc/fsutil-fs-in-opt"

This reverts commit c62e704, reversing
changes made to d5c1d78.

Signed-off-by: CrazyMax <[email protected]>

cache/util/fsutil.go

@@ -90,17 +90,17 @@ type ReadDirRequest struct {
 func ReadDir(ctx context.Context, mount snapshot.Mountable, req ReadDirRequest) ([]*fstypes.Stat, error) {
 	var (
 		rd []*fstypes.Stat
-		fo fsutil.FilterOpt
+		wo fsutil.WalkOpt
 	)
 	if req.IncludePattern != "" {
-		fo.IncludePatterns = append(fo.IncludePatterns, req.IncludePattern)
+		wo.IncludePatterns = append(wo.IncludePatterns, req.IncludePattern)
 	}
 	err := withMount(ctx, mount, func(root string) error {
 		fp, err := fs.RootPath(root, req.Path)
 		if err != nil {
 			return errors.WithStack(err)
 		}
-		return fsutil.Walk(ctx, fp, &fo, func(path string, info os.FileInfo, err error) error {
+		return fsutil.Walk(ctx, fp, &wo, func(path string, info os.FileInfo, err error) error {
 			if err != nil {
 				return errors.Wrapf(err, "walking %q", root)
 			}

client/solve.go

@@ -35,8 +35,7 @@ import (
 
 type SolveOpt struct {
 	Exports               []ExportEntry
-	LocalDirs             map[string]string // Deprecated: use LocalMounts
-	LocalMounts           map[string]fsutil.FS
+	LocalDirs             map[string]string
 	OCIStores             map[string]content.Store
 	SharedKey             string
 	Frontend              string
@@ -91,11 +90,7 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		return nil, errors.New("invalid with def and cb")
 	}
 
-	mounts, err := prepareMounts(&opt)
-	if err != nil {
-		return nil, err
-	}
-	syncedDirs, err := prepareSyncedFiles(def, mounts)
+	syncedDirs, err := prepareSyncedDirs(def, opt.LocalDirs)
 	if err != nil {
 		return nil, err
 	}
@@ -366,23 +361,26 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 	return res, nil
 }
 
-func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (filesync.StaticDirSource, error) {
+func prepareSyncedDirs(def *llb.Definition, localDirs map[string]string) (filesync.StaticDirSource, error) {
+	for _, d := range localDirs {
+		fi, err := os.Stat(d)
+		if err != nil {
+			return nil, errors.Wrapf(err, "could not find %s", d)
+		}
+		if !fi.IsDir() {
+			return nil, errors.Errorf("%s not a directory", d)
+		}
+	}
 	resetUIDAndGID := func(p string, st *fstypes.Stat) fsutil.MapResult {
 		st.Uid = 0
 		st.Gid = 0
 		return fsutil.MapResultKeep
 	}
 
-	result := make(filesync.StaticDirSource, len(localMounts))
+	dirs := make(filesync.StaticDirSource, len(localDirs))
 	if def == nil {
-		for name, mount := range localMounts {
-			mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
-				Map: resetUIDAndGID,
-			})
-			if err != nil {
-				return nil, err
-			}
-			result[name] = mount
+		for name, d := range localDirs {
+			dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
 		}
 	} else {
 		for _, dt := range def.Def {
@@ -393,22 +391,16 @@ func prepareSyncedFiles(def *llb.Definition, localMounts map[string]fsutil.FS) (
 			if src := op.GetSource(); src != nil {
 				if strings.HasPrefix(src.Identifier, "local://") {
 					name := strings.TrimPrefix(src.Identifier, "local://")
-					mount, ok := localMounts[name]
+					d, ok := localDirs[name]
 					if !ok {
 						return nil, errors.Errorf("local directory %s not enabled", name)
 					}
-					mount, err := fsutil.NewFilterFS(mount, &fsutil.FilterOpt{
-						Map: resetUIDAndGID,
-					})
-					if err != nil {
-						return nil, err
-					}
-					result[name] = mount
+					dirs[name] = filesync.SyncedDir{Dir: d, Map: resetUIDAndGID}
 				}
 			}
 		}
 	}
-	return result, nil
+	return dirs, nil
 }
 
 func defaultSessionName() string {
@@ -531,22 +523,3 @@ func parseCacheOptions(ctx context.Context, isGateway bool, opt SolveOpt) (*cach
 	}
 	return &res, nil
 }
-
-func prepareMounts(opt *SolveOpt) (map[string]fsutil.FS, error) {
-	// merge local mounts and fallback local directories together
-	mounts := make(map[string]fsutil.FS)
-	for k, mount := range opt.LocalMounts {
-		mounts[k] = mount
-	}
-	for k, dir := range opt.LocalDirs {
-		mount, err := fsutil.NewFS(dir)
-		if err != nil {
-			return nil, err
-		}
-		if _, ok := mounts[k]; ok {
-			return nil, errors.Errorf("local mount %s already exists", k)
-		}
-		mounts[k] = mount
-	}
-	return mounts, nil
-}

exporter/local/export.go

@@ -109,8 +109,8 @@ func (e *localExporterInstance) Export(ctx context.Context, inp *exporter.Source
 
 			if !e.opts.PlatformSplit {
 				// check for duplicate paths
-				err = outputFS.Walk(ctx, "", func(p string, entry os.DirEntry, err error) error {
-					if entry.IsDir() {
+				err = outputFS.Walk(ctx, func(p string, fi os.FileInfo, err error) error {
+					if fi.IsDir() {
 						return nil
 					}
 					if err != nil && !errors.Is(err, os.ErrNotExist) {

exporter/local/fs.go

@@ -98,14 +98,9 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
 		cleanup = lm.Unmount
 	}
 
-	outputFS, err := fsutil.NewFS(src)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	// wrap the output filesystem, applying appropriate filters
-	filterOpt := &fsutil.FilterOpt{}
+	walkOpt := &fsutil.WalkOpt{}
 	var idMapFunc func(p string, st *fstypes.Stat) fsutil.MapResult
+
 	if idmap != nil {
 		idMapFunc = func(p string, st *fstypes.Stat) fsutil.MapResult {
 			uid, gid, err := idmap.ToContainer(idtools.Identity{
@@ -120,23 +115,19 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
 			return fsutil.MapResultKeep
 		}
 	}
-	filterOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
+
+	walkOpt.Map = func(p string, st *fstypes.Stat) fsutil.MapResult {
 		res := fsutil.MapResultKeep
 		if idMapFunc != nil {
-			// apply host uid/gid
 			res = idMapFunc(p, st)
 		}
 		if opt.Epoch != nil {
-			// apply used-specified epoch time
 			st.ModTime = opt.Epoch.UnixNano()
 		}
 		return res
 	}
-	outputFS, err = fsutil.NewFilterFS(outputFS, filterOpt)
-	if err != nil {
-		return nil, nil, err
-	}
 
+	outputFS := fsutil.NewFS(src, walkOpt)
 	attestations = attestation.Filter(attestations, nil, map[string][]byte{
 		result.AttestationInlineOnlyKey: []byte(strconv.FormatBool(true)),
 	})
@@ -146,11 +137,11 @@ func CreateFS(ctx context.Context, sessionID string, k string, ref cache.Immutab
 	}
 	if len(attestations) > 0 {
 		subjects := []intoto.Subject{}
-		err = outputFS.Walk(ctx, "", func(path string, entry fs.DirEntry, err error) error {
+		err = outputFS.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
 			if err != nil {
 				return err
 			}
-			if !entry.Type().IsRegular() {
+			if !info.Mode().IsRegular() {
 				return nil
 			}
 			f, err := outputFS.Open(path)

go.mod

@@ -64,7 +64,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spdx/tools-golang v0.5.1
 	github.com/stretchr/testify v1.8.4
-	github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302
+	github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb
 	github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7
 	github.com/tonistiigi/go-archvariant v1.0.0
 	github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea

go.sum

@@ -1204,8 +1204,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1
 github.com/tommy-muehle/go-mnd v1.1.1/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/tommy-muehle/go-mnd v1.3.1-0.20200224220436-e6f9a994e8fa/go.mod h1:dSUh0FtTP8VhvkL1S+gUR1OKd9ZnSaozuI6r3m6wOig=
 github.com/tonistiigi/fsutil v0.0.0-20201103201449-0834f99b7b85/go.mod h1:a7cilN64dG941IOXfhJhlH0qB92hxJ9A1ewrdUmJ6xo=
-github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302 h1:ZT8ibgassurSISJ1Pj26NsM3vY2jxFZn63Nd/TpHmRw=
-github.com/tonistiigi/fsutil v0.0.0-20230825212630-f09800878302/go.mod h1:9kMVqMyQ/Sx2df5LtnGG+nbrmiZzCS7V6gjW3oGHsvI=
+github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb h1:uUe8rNyVXM8moActoBol6Xf6xX2GMr7SosR2EywMvGg=
+github.com/tonistiigi/fsutil v0.0.0-20230629203738-36ef4d8c0dbb/go.mod h1:SxX/oNQ/ag6Vaoli547ipFK9J7BZn5JqJG0JE8lf8bA=
 github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7 h1:8eY6m1mjgyB8XySUR7WvebTM8D/Vs86jLJzD/Tw7zkc=
 github.com/tonistiigi/go-actions-cache v0.0.0-20220404170428-0bdeb6e1eac7/go.mod h1:qqvyZqkfwkoJuPU/bw61bItaoO0SJ8YSW0vSVRRvsRg=
 github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0=

session/filesync/filesync.go

@@ -35,15 +35,20 @@ type fsSyncProvider struct {
 	doneCh chan error
 }
 
+type SyncedDir struct {
+	Dir string
+	Map func(string, *fstypes.Stat) fsutil.MapResult
+}
+
 type DirSource interface {
-	LookupDir(string) (fsutil.FS, bool)
+	LookupDir(string) (SyncedDir, bool)
 }
 
-type StaticDirSource map[string]fsutil.FS
+type StaticDirSource map[string]SyncedDir
 
 var _ DirSource = StaticDirSource{}
 
-func (dirs StaticDirSource) LookupDir(name string) (fsutil.FS, bool) {
+func (dirs StaticDirSource) LookupDir(name string) (SyncedDir, bool) {
 	dir, found := dirs[name]
 	return dir, found
 }
@@ -87,22 +92,15 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
 		dirName = name[0]
 	}
 
-	excludes := opts[keyExcludePatterns]
-	includes := opts[keyIncludePatterns]
-	followPaths := opts[keyFollowPaths]
-
 	dir, ok := sp.dirs.LookupDir(dirName)
 	if !ok {
 		return InvalidSessionError{status.Errorf(codes.NotFound, "no access allowed to dir %q", dirName)}
 	}
-	dir, err := fsutil.NewFilterFS(dir, &fsutil.FilterOpt{
-		ExcludePatterns: excludes,
-		IncludePatterns: includes,
-		FollowPaths:     followPaths,
-	})
-	if err != nil {
-		return err
-	}
+
+	excludes := opts[keyExcludePatterns]
+	includes := opts[keyIncludePatterns]
+
+	followPaths := opts[keyFollowPaths]
 
 	var progress progressCb
 	if sp.p != nil {
@@ -115,7 +113,12 @@ func (sp *fsSyncProvider) handle(method string, stream grpc.ServerStream) (retEr
 		doneCh = sp.doneCh
 		sp.doneCh = nil
 	}
-	err = pr.sendFn(stream, dir, progress)
+	err := pr.sendFn(stream, fsutil.NewFS(dir.Dir, &fsutil.WalkOpt{
+		ExcludePatterns: excludes,
+		IncludePatterns: includes,
+		FollowPaths:     followPaths,
+		Map:             dir.Map,
+	}), progress)
 	if doneCh != nil {
 		if err != nil {
 			doneCh <- err

session/filesync/filesync_test.go

@@ -10,7 +10,6 @@ import (
 	"github.com/moby/buildkit/session/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/tonistiigi/fsutil"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -19,11 +18,9 @@ func TestFileSyncIncludePatterns(t *testing.T) {
 	t.Parallel()
 
 	tmpDir := t.TempDir()
-	tmpFS, err := fsutil.NewFS(tmpDir)
-	require.NoError(t, err)
 	destDir := t.TempDir()
 
-	err = os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600)
+	err := os.WriteFile(filepath.Join(tmpDir, "foo"), []byte("content1"), 0600)
 	require.NoError(t, err)
 
 	err = os.WriteFile(filepath.Join(tmpDir, "bar"), []byte("content2"), 0600)
@@ -35,7 +32,7 @@ func TestFileSyncIncludePatterns(t *testing.T) {
 	m, err := session.NewManager()
 	require.NoError(t, err)
 
-	fs := NewFSSyncProvider(StaticDirSource{"test0": tmpFS})
+	fs := NewFSSyncProvider(StaticDirSource{"test0": {Dir: tmpDir}})
 	s.Allow(fs)
 
 	dialer := session.Dialer(testutil.TestStream(testutil.Handler(m.HandleConn)))

util/staticfs/merge.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/fs"
 	"os"
+	"path/filepath"
 
 	"github.com/tonistiigi/fsutil"
 	"golang.org/x/sync/errgroup"
@@ -25,9 +26,9 @@ func NewMergeFS(lower, upper fsutil.FS) *MergeFS {
 }
 
 type record struct {
-	path  string
-	entry fs.DirEntry
-	err   error
+	path string
+	fi   fs.FileInfo
+	err  error
 }
 
 func (r *record) key() string {
@@ -37,26 +38,26 @@ func (r *record) key() string {
 	return convertPathToKey(r.path)
 }
 
-func (mfs *MergeFS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc) error {
+func (mfs *MergeFS) Walk(ctx context.Context, fn filepath.WalkFunc) error {
 	ch1 := make(chan *record, 10)
 	ch2 := make(chan *record, 10)
 
 	eg, ctx := errgroup.WithContext(ctx)
 	eg.Go(func() error {
 		defer close(ch1)
-		return mfs.Lower.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error {
+		return mfs.Lower.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
 			select {
-			case ch1 <- &record{path: path, entry: entry, err: err}:
+			case ch1 <- &record{path: path, fi: info, err: err}:
 			case <-ctx.Done():
 			}
 			return context.Cause(ctx)
 		})
 	})
 	eg.Go(func() error {
 		defer close(ch2)
-		return mfs.Upper.Walk(ctx, target, func(path string, entry fs.DirEntry, err error) error {
+		return mfs.Upper.Walk(ctx, func(path string, info fs.FileInfo, err error) error {
 			select {
-			case ch2 <- &record{path: path, entry: entry, err: err}:
+			case ch2 <- &record{path: path, fi: info, err: err}:
 			case <-ctx.Done():
 			}
 			return context.Cause(ctx)
@@ -74,13 +75,13 @@ func (mfs *MergeFS) Walk(ctx context.Context, target string, fn fs.WalkDirFunc)
 				break
 			}
 			if !ok2 || ok1 && key1 < key2 {
-				if err := fn(next1.path, next1.entry, next1.err); err != nil {
+				if err := fn(next1.path, next1.fi, next1.err); err != nil {
 					return err
 				}
 				next1, ok1 = <-ch1
 				key1 = next1.key()
 			} else if !ok1 || ok2 && key1 >= key2 {
-				if err := fn(next2.path, next2.entry, next2.err); err != nil {
+				if err := fn(next2.path, next2.fi, next2.err); err != nil {
 					return err
 				}
 				if ok1 && key1 == key2 {