From f293670e36b1de9e05085a18577f52542d9bb424 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Thu, 22 Apr 2021 14:32:06 -0700 Subject: [PATCH] Cache awareness of IncludePatterns/ExcludePatterns Consider IncludePatterns and ExcludePattern when calculating content hashes. --- cache/contenthash/checksum.go | 172 +++++++++++++++------- cache/contenthash/checksum_test.go | 222 ++++++++++++++++++++--------- solver/llbsolver/ops/file.go | 43 +++--- solver/llbsolver/result.go | 37 +++-- 4 files changed, 318 insertions(+), 156 deletions(-) diff --git a/cache/contenthash/checksum.go b/cache/contenthash/checksum.go index ac9d3ec36a271..6eeebf9c1d854 100644 --- a/cache/contenthash/checksum.go +++ b/cache/contenthash/checksum.go @@ -8,8 +8,10 @@ import ( "os" "path" "path/filepath" + "strings" "sync" + "github.com/docker/docker/pkg/fileutils" "github.com/docker/docker/pkg/idtools" iradix "github.com/hashicorp/go-immutable-radix" "github.com/hashicorp/golang-lru/simplelru" @@ -21,6 +23,7 @@ import ( digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/tonistiigi/fsutil" + "github.com/tonistiigi/fsutil/prefix" fstypes "github.com/tonistiigi/fsutil/types" ) @@ -45,12 +48,15 @@ func getDefaultManager() *cacheManager { // header, "/dir" is for contents. For the root node "" (empty string) is the // key for root, "/" for the root header -func Checksum(ctx context.Context, ref cache.ImmutableRef, path string, followLinks bool, s session.Group) (digest.Digest, error) { - return getDefaultManager().Checksum(ctx, ref, path, followLinks, s) +type ChecksumOpts struct { + FollowLinks bool + Wildcard bool + IncludePatterns []string + ExcludePatterns []string } -func ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, path string, followLinks bool, s session.Group) (digest.Digest, error) { - return getDefaultManager().ChecksumWildcard(ctx, ref, path, followLinks, s) +func Checksum(ctx context.Context, ref cache.ImmutableRef, path string, opts ChecksumOpts, s session.Group) (digest.Digest, error) { + return getDefaultManager().Checksum(ctx, ref, path, opts, s) } func GetCacheContext(ctx context.Context, md *metadata.StorageItem, idmap *idtools.IdentityMapping) (CacheContext, error) { @@ -66,8 +72,7 @@ func ClearCacheContext(md *metadata.StorageItem) { } type CacheContext interface { - Checksum(ctx context.Context, ref cache.Mountable, p string, followLinks bool, s session.Group) (digest.Digest, error) - ChecksumWildcard(ctx context.Context, ref cache.Mountable, p string, followLinks bool, s session.Group) (digest.Digest, error) + Checksum(ctx context.Context, ref cache.Mountable, p string, opts ChecksumOpts, s session.Group) (digest.Digest, error) HandleChange(kind fsutil.ChangeKind, p string, fi os.FileInfo, err error) error } @@ -75,7 +80,7 @@ type Hashed interface { Digest() digest.Digest } -type Wildcard struct { +type IncludedPath struct { Path string Record *CacheRecord } @@ -86,20 +91,12 @@ type cacheManager struct { lruMu sync.Mutex } -func (cm *cacheManager) Checksum(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool, s session.Group) (digest.Digest, error) { +func (cm *cacheManager) Checksum(ctx context.Context, ref cache.ImmutableRef, p string, opts ChecksumOpts, s session.Group) (digest.Digest, error) { cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata()), ref.IdentityMapping()) if err != nil { return "", nil } - return cc.Checksum(ctx, ref, p, followLinks, s) -} - -func (cm *cacheManager) ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool, s session.Group) (digest.Digest, error) { - cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata()), ref.IdentityMapping()) - if err != nil { - return "", nil - } - return cc.ChecksumWildcard(ctx, ref, p, followLinks, s) + return cc.Checksum(ctx, ref, p, opts, s) } func (cm *cacheManager) GetCacheContext(ctx context.Context, md *metadata.StorageItem, idmap *idtools.IdentityMapping) (CacheContext, error) { @@ -264,12 +261,17 @@ func (cc *cacheContext) save() error { return cc.md.SetExternal(keyContentHash, dt) } -// HandleChange notifies the source about a modification operation -func (cc *cacheContext) HandleChange(kind fsutil.ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { +func keyPath(p string) string { p = path.Join("/", filepath.ToSlash(p)) if p == "/" { p = "" } + return p +} + +// HandleChange notifies the source about a modification operation +func (cc *cacheContext) HandleChange(kind fsutil.ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { + p = keyPath(p) k := convertPathToKey([]byte(p)) deleteDir := func(cr *CacheRecord) { @@ -382,36 +384,40 @@ func (cc *cacheContext) HandleChange(kind fsutil.ChangeKind, p string, fi os.Fil return nil } -func (cc *cacheContext) ChecksumWildcard(ctx context.Context, mountable cache.Mountable, p string, followLinks bool, s session.Group) (digest.Digest, error) { +func (cc *cacheContext) Checksum(ctx context.Context, mountable cache.Mountable, p string, opts ChecksumOpts, s session.Group) (digest.Digest, error) { m := &mount{mountable: mountable, session: s} defer m.clean() - wildcards, err := cc.wildcards(ctx, m, p) + if !opts.Wildcard && len(opts.IncludePatterns) == 0 && len(opts.ExcludePatterns) == 0 { + return cc.checksumFollow(ctx, m, p, opts.FollowLinks) + } + + includedPaths, err := cc.includedPaths(ctx, m, p, opts) if err != nil { return "", err } - if followLinks { - for i, w := range wildcards { + if opts.FollowLinks { + for i, w := range includedPaths { if w.Record.Type == CacheRecordTypeSymlink { - dgst, err := cc.checksumFollow(ctx, m, w.Path, followLinks) + dgst, err := cc.checksumFollow(ctx, m, w.Path, opts.FollowLinks) if err != nil { return "", err } - wildcards[i].Record = &CacheRecord{Digest: dgst} + includedPaths[i].Record = &CacheRecord{Digest: dgst} } } } - if len(wildcards) == 0 { + if len(includedPaths) == 0 { return digest.FromBytes([]byte{}), nil } - if len(wildcards) == 1 && path.Base(p) == path.Base(wildcards[0].Path) { - return wildcards[0].Record.Digest, nil + if len(includedPaths) == 1 && path.Base(p) == path.Base(includedPaths[0].Path) { + return includedPaths[0].Record.Digest, nil } digester := digest.Canonical.Digester() - for i, w := range wildcards { + for i, w := range includedPaths { if i != 0 { digester.Hash().Write([]byte{0}) } @@ -421,13 +427,6 @@ func (cc *cacheContext) ChecksumWildcard(ctx context.Context, mountable cache.Mo return digester.Digest(), nil } -func (cc *cacheContext) Checksum(ctx context.Context, mountable cache.Mountable, p string, followLinks bool, s session.Group) (digest.Digest, error) { - m := &mount{mountable: mountable, session: s} - defer m.clean() - - return cc.checksumFollow(ctx, m, p, followLinks) -} - func (cc *cacheContext) checksumFollow(ctx context.Context, m *mount, p string, follow bool) (digest.Digest, error) { const maxSymlinkLimit = 255 i := 0 @@ -452,7 +451,7 @@ func (cc *cacheContext) checksumFollow(ctx context.Context, m *mount, p string, } } -func (cc *cacheContext) wildcards(ctx context.Context, m *mount, p string) ([]*Wildcard, error) { +func (cc *cacheContext) includedPaths(ctx context.Context, m *mount, p string, opts ChecksumOpts) ([]*IncludedPath, error) { cc.mu.Lock() defer cc.mu.Unlock() @@ -478,18 +477,34 @@ func (cc *cacheContext) wildcards(ctx context.Context, m *mount, p string) ([]*W } }() - p = path.Join("/", filepath.ToSlash(p)) - if p == "/" { - p = "" + p = keyPath(p) + + rootedIncludePatterns := make([]string, len(opts.IncludePatterns)) + for i, includePattern := range opts.IncludePatterns { + rootedIncludePatterns[i] = keyPath(includePattern) + } + + var excludePatternMatcher *fileutils.PatternMatcher + if len(opts.ExcludePatterns) != 0 { + rootedExcludePatterns := make([]string, len(opts.ExcludePatterns)) + for i, excludePattern := range opts.ExcludePatterns { + rootedExcludePatterns[i] = keyPath(excludePattern) + } + excludePatternMatcher, err = fileutils.NewPatternMatcher(rootedExcludePatterns) + if err != nil { + return nil, errors.Wrapf(err, "invalid excludepatterns: %s", opts.ExcludePatterns) + } } - wildcards := make([]*Wildcard, 0, 2) + includedPaths := make([]*IncludedPath, 0, 2) txn := cc.tree.Txn() root = txn.Root() var updated bool + lastIncludedDir := "" iter := root.Seek([]byte{}) +treeWalk: for { k, _, ok := iter.Next() if !ok { @@ -498,8 +513,8 @@ func (cc *cacheContext) wildcards(ctx context.Context, m *mount, p string) ([]*W if len(k) > 0 && k[len(k)-1] == byte(0) { continue } - fn := convertKeyToPath(k) - b, err := path.Match(p, string(fn)) + fn := string(convertKeyToPath(k)) + b, err := shouldIncludePath(p, fn, opts.Wildcard, rootedIncludePatterns, excludePatternMatcher, lastIncludedDir) if err != nil { return nil, err } @@ -515,8 +530,25 @@ func (cc *cacheContext) wildcards(ctx context.Context, m *mount, p string) ([]*W updated = true } - wildcards = append(wildcards, &Wildcard{Path: string(fn), Record: cr}) + if cr.Type == CacheRecordTypeDir { + lastIncludedDir = fn + + if excludePatternMatcher != nil { + dirSlash := fn + "/" + for _, pat := range excludePatternMatcher.Patterns() { + patStr := pat.String() + "/" + if strings.HasPrefix(patStr, dirSlash) { + // This dir has exclusions underneath it. Do not + // include the dir as a whole. Instead, continue + // walking and only include paths that match the + // filters. + continue treeWalk + } + } + } + } + includedPaths = append(includedPaths, &IncludedPath{Path: fn, Record: cr}) if cr.Type == CacheRecordTypeDir { iter = root.Seek(append(k, 0, 0xff)) } @@ -525,15 +557,57 @@ func (cc *cacheContext) wildcards(ctx context.Context, m *mount, p string) ([]*W cc.tree = txn.Commit() cc.dirty = updated - return wildcards, nil + return includedPaths, nil } -func (cc *cacheContext) checksumNoFollow(ctx context.Context, m *mount, p string) (*CacheRecord, error) { - p = path.Join("/", filepath.ToSlash(p)) - if p == "/" { - p = "" +func shouldIncludePath( + p string, + candidate string, + wildcard bool, + rootedIncludePatterns []string, + excludePatternMatcher *fileutils.PatternMatcher, + lastIncludedDir string, +) (bool, error) { + if wildcard { + include, err := path.Match(p, candidate) + if err != nil { + return include, err + } + if !include { + return false, nil + } + } else if !strings.HasPrefix(candidate+"/", p+"/") { + return false, nil } + if len(rootedIncludePatterns) != 0 && + (lastIncludedDir == "" || + !strings.HasPrefix(candidate, lastIncludedDir+"/")) { + matched := false + for _, pattern := range rootedIncludePatterns { + if ok, partial := prefix.Match(pattern, candidate); ok && !partial { + matched = true + break + } + } + if !matched { + return false, nil + } + } + + if excludePatternMatcher == nil { + return true, nil + } + m, err := excludePatternMatcher.Matches(candidate) + if err != nil { + return false, errors.Wrap(err, "failed to match excludepatterns") + } + return !m, nil +} + +func (cc *cacheContext) checksumNoFollow(ctx context.Context, m *mount, p string) (*CacheRecord, error) { + p = keyPath(p) + cc.mu.RLock() if cc.txn == nil { root := cc.tree.Root() diff --git a/cache/contenthash/checksum_test.go b/cache/contenthash/checksum_test.go index 0c377707ee263..d8100b49a1a62 100644 --- a/cache/contenthash/checksum_test.go +++ b/cache/contenthash/checksum_test.go @@ -60,7 +60,7 @@ func TestChecksumSymlinkNoParentScan(t *testing.T) { cc, err := newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - dgst, err := cc.Checksum(context.TODO(), ref, "aa/ln/bb/cc/dd", true, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "aa/ln/bb/cc/dd", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) } @@ -88,15 +88,15 @@ func TestChecksumHardlinks(t *testing.T) { cc, err := newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - dgst, err := cc.Checksum(context.TODO(), ref, "abc/foo", false, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "abc/foo", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = cc.Checksum(context.TODO(), ref, "ln", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "ln", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = cc.Checksum(context.TODO(), ref, "ln2", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "ln2", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -109,15 +109,15 @@ func TestChecksumHardlinks(t *testing.T) { err = emit(cc2.HandleChange, changeStream(ch)) require.NoError(t, err) - dgst, err = cc2.Checksum(context.TODO(), ref, "abc/foo", false, nil) + dgst, err = cc2.Checksum(context.TODO(), ref, "abc/foo", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = cc2.Checksum(context.TODO(), ref, "ln", false, nil) + dgst, err = cc2.Checksum(context.TODO(), ref, "ln", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = cc2.Checksum(context.TODO(), ref, "ln2", false, nil) + dgst, err = cc2.Checksum(context.TODO(), ref, "ln2", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -135,20 +135,20 @@ func TestChecksumHardlinks(t *testing.T) { data1Expected := "sha256:c2b5e234f5f38fc5864da7def04782f82501a40d46192e4207d5b3f0c3c4732b" - dgst, err = cc2.Checksum(context.TODO(), ref, "abc/foo", false, nil) + dgst, err = cc2.Checksum(context.TODO(), ref, "abc/foo", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, data1Expected, string(dgst)) - dgst, err = cc2.Checksum(context.TODO(), ref, "ln", false, nil) + dgst, err = cc2.Checksum(context.TODO(), ref, "ln", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, data1Expected, string(dgst)) - dgst, err = cc2.Checksum(context.TODO(), ref, "ln2", false, nil) + dgst, err = cc2.Checksum(context.TODO(), ref, "ln2", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) } -func TestChecksumWildcard(t *testing.T) { +func TestChecksumWildcardOrFilter(t *testing.T) { t.Parallel() tmpdir, err := ioutil.TempDir("", "buildkit-state") require.NoError(t, err) @@ -177,27 +177,27 @@ func TestChecksumWildcard(t *testing.T) { cc, err := newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - dgst, err := cc.ChecksumWildcard(context.TODO(), ref, "f*o", false, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "f*o", ChecksumOpts{Wildcard: true}, nil) require.NoError(t, err) require.Equal(t, digest.FromBytes(append([]byte("foo"), []byte(dgstFileData0)...)), dgst) expFoos := digest.Digest("sha256:7f51c821895cfc116d3f64231dfb438e87a237ecbbe027cd96b7ee5e763cc569") - dgst, err = cc.ChecksumWildcard(context.TODO(), ref, "f*", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "f*", ChecksumOpts{Wildcard: true}, nil) require.NoError(t, err) require.Equal(t, expFoos, dgst) - dgst, err = cc.ChecksumWildcard(context.TODO(), ref, "x/d?", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "x/d?", ChecksumOpts{Wildcard: true}, nil) require.NoError(t, err) require.Equal(t, digest.FromBytes(append([]byte("d0"), []byte(dgstDirD0)...)), dgst) - dgst, err = cc.ChecksumWildcard(context.TODO(), ref, "x/d?/def", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "x/d?/def", ChecksumOpts{FollowLinks: true, Wildcard: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) expFoos2 := digest.Digest("sha256:8afc09c7018d65d5eb318a9ef55cb704dec1f06d288181d913fc27a571aa042d") - dgst, err = cc.ChecksumWildcard(context.TODO(), ref, "y*", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "y*", ChecksumOpts{FollowLinks: true, Wildcard: true}, nil) require.NoError(t, err) require.Equal(t, expFoos2, dgst) @@ -221,7 +221,7 @@ func TestChecksumWildcardWithBadMountable(t *testing.T) { cc, err := newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - _, err = cc.ChecksumWildcard(context.TODO(), newBadMountable(), "*", false, nil) + _, err = cc.Checksum(context.TODO(), newBadMountable(), "*", ChecksumOpts{Wildcard: true}, nil) require.Error(t, err) } @@ -252,31 +252,31 @@ func TestSymlinksNoFollow(t *testing.T) { expectedSym := digest.Digest("sha256:a2ba571981f48ec34eb79c9a3ab091b6491e825c2f7e9914ea86e8e958be7fae") - dgst, err := cc.ChecksumWildcard(context.TODO(), ref, "sym", false, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "sym", ChecksumOpts{Wildcard: true}, nil) require.NoError(t, err) require.Equal(t, expectedSym, dgst) - dgst, err = cc.ChecksumWildcard(context.TODO(), ref, "sym2", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "sym2", ChecksumOpts{Wildcard: true}, nil) require.NoError(t, err) require.NotEqual(t, expectedSym, dgst) - dgst, err = cc.ChecksumWildcard(context.TODO(), ref, "foo/ghi", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "foo/ghi", ChecksumOpts{Wildcard: true}, nil) require.NoError(t, err) require.Equal(t, expectedSym, dgst) - _, err = cc.ChecksumWildcard(context.TODO(), ref, "foo/ghi", true, nil) // same because broken symlink + _, err = cc.Checksum(context.TODO(), ref, "foo/ghi", ChecksumOpts{FollowLinks: true, Wildcard: true}, nil) // same because broken symlink require.Error(t, err) require.Equal(t, true, errors.Is(err, errNotFound)) - _, err = cc.ChecksumWildcard(context.TODO(), ref, "y1", true, nil) + _, err = cc.Checksum(context.TODO(), ref, "y1", ChecksumOpts{FollowLinks: true, Wildcard: true}, nil) require.Error(t, err) require.Equal(t, true, errors.Is(err, errNotFound)) - dgst, err = cc.Checksum(context.TODO(), ref, "sym", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "sym", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, expectedSym, dgst) - dgst, err = cc.Checksum(context.TODO(), ref, "foo/ghi", false, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "foo/ghi", ChecksumOpts{}, nil) require.NoError(t, err) require.Equal(t, expectedSym, dgst) @@ -312,48 +312,48 @@ func TestChecksumBasicFile(t *testing.T) { cc, err := newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - _, err = cc.Checksum(context.TODO(), ref, "nosuch", true, nil) + _, err = cc.Checksum(context.TODO(), ref, "nosuch", ChecksumOpts{FollowLinks: true}, nil) require.Error(t, err) - dgst, err := cc.Checksum(context.TODO(), ref, "foo", true, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) // second file returns different hash - dgst, err = cc.Checksum(context.TODO(), ref, "bar", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "bar", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, digest.Digest("sha256:c2b5e234f5f38fc5864da7def04782f82501a40d46192e4207d5b3f0c3c4732b"), dgst) // same file inside a directory - dgst, err = cc.Checksum(context.TODO(), ref, "d0/abc", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d0/abc", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) // repeat because codepath is different - dgst, err = cc.Checksum(context.TODO(), ref, "d0/abc", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d0/abc", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) // symlink to the same file is followed, returns same hash - dgst, err = cc.Checksum(context.TODO(), ref, "d0/def", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d0/def", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - _, err = cc.Checksum(context.TODO(), ref, "d0/ghi", true, nil) + _, err = cc.Checksum(context.TODO(), ref, "d0/ghi", ChecksumOpts{FollowLinks: true}, nil) require.Error(t, err) require.Equal(t, true, errors.Is(err, errNotFound)) - dgst, err = cc.Checksum(context.TODO(), ref, "/", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "/", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, digest.Digest("sha256:427c9cf9ae98c0f81fb57a3076b965c7c149b6b0a85625ad4e884236649a42c6"), dgst) - dgst, err = cc.Checksum(context.TODO(), ref, "d0", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d0", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstDirD0, dgst) @@ -373,7 +373,7 @@ func TestChecksumBasicFile(t *testing.T) { cc, err = newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - dgst, err = cc.Checksum(context.TODO(), ref, "/", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "/", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstDirD0, dgst) @@ -392,7 +392,7 @@ func TestChecksumBasicFile(t *testing.T) { cc, err = newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - dgst, err = cc.Checksum(context.TODO(), ref, "/", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "/", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstDirD0Modified, dgst) @@ -418,14 +418,14 @@ func TestChecksumBasicFile(t *testing.T) { cc, err = newCacheContext(ref.Metadata(), nil) require.NoError(t, err) - dgst, err = cc.Checksum(context.TODO(), ref, "abc/aa/foo", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "abc/aa/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, digest.Digest("sha256:1c67653c3cf95b12a0014e2c4cd1d776b474b3218aee54155d6ae27b9b999c54"), dgst) require.NotEqual(t, dgstDirD0, dgst) // this will force rescan - dgst, err = cc.Checksum(context.TODO(), ref, "d0", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d0", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstDirD0, dgst) @@ -434,6 +434,92 @@ func TestChecksumBasicFile(t *testing.T) { require.NoError(t, err) } +func TestChecksumIncludeExclude(t *testing.T) { + t.Parallel() + tmpdir, err := ioutil.TempDir("", "buildkit-state") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) + require.NoError(t, err) + cm, _ := setupCacheManager(t, tmpdir, "native", snapshotter) + defer cm.Close() + + ch := []string{ + "ADD foo file data0", + "ADD bar file data1", + "ADD d0 dir", + "ADD d0/abc file abc", + "ADD d1 dir", + "ADD d1/def file def", + } + + ref := createRef(t, cm, ch) + + cc, err := newCacheContext(ref.Metadata(), nil) + require.NoError(t, err) + + dgst, err := cc.Checksum(context.TODO(), ref, "foo", ChecksumOpts{IncludePatterns: []string{"foo"}}, nil) + require.NoError(t, err) + + require.Equal(t, dgstFileData0, dgst) + + dgstFoo, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"foo"}}, nil) + require.NoError(t, err) + dgstFooBar, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"foo", "bar"}}, nil) + require.NoError(t, err) + + require.NotEqual(t, dgstFoo, dgstFooBar) + + dgstD0, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"d0/*"}}, nil) + require.NoError(t, err) + dgstD1, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"d1/*"}}, nil) + require.NoError(t, err) + + err = ref.Release(context.TODO()) + require.NoError(t, err) + + // add some files + ch = []string{ + "ADD foo file data0", + "ADD bar file data1", + "ADD baz file data2", + "ADD d0 dir", + "ADD d0/abc file abc", + "ADD d0/xyz file xyz", + "ADD d1 dir", + "ADD d1/def file def", + } + + ref = createRef(t, cm, ch) + + cc, err = newCacheContext(ref.Metadata(), nil) + require.NoError(t, err) + + dgstFoo2, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"foo"}}, nil) + require.NoError(t, err) + dgstFooBar2, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"foo", "bar"}}, nil) + require.NoError(t, err) + + require.Equal(t, dgstFoo, dgstFoo2) + require.Equal(t, dgstFooBar, dgstFooBar2) + + dgstD02, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"d0/*"}}, nil) + require.NoError(t, err) + require.NotEqual(t, dgstD0, dgstD02) + + dgstD12, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"d1/*"}}, nil) + require.NoError(t, err) + require.Equal(t, dgstD1, dgstD12) + + dgstD0Exclude, err := cc.Checksum(context.TODO(), ref, "", ChecksumOpts{IncludePatterns: []string{"d0/*"}, ExcludePatterns: []string{"d0/xyz"}}, nil) + require.NoError(t, err) + require.Equal(t, dgstD0, dgstD0Exclude) + + err = ref.Release(context.TODO()) + require.NoError(t, err) +} + func TestHandleChange(t *testing.T) { t.Parallel() tmpdir, err := ioutil.TempDir("", "buildkit-state") @@ -465,19 +551,19 @@ func TestHandleChange(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - dgstFoo, err := cc.Checksum(context.TODO(), ref, "foo", true, nil) + dgstFoo, err := cc.Checksum(context.TODO(), ref, "foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgstFoo) // symlink to the same file is followed, returns same hash - dgst, err := cc.Checksum(context.TODO(), ref, "d0/def", true, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "d0/def", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFoo, dgst) // symlink to the same file is followed, returns same hash - dgst, err = cc.Checksum(context.TODO(), ref, "d0", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d0", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstDirD0, dgst) @@ -489,7 +575,7 @@ func TestHandleChange(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - dgst, err = cc.Checksum(context.TODO(), ref, "d0", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d0", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstDirD0Modified, dgst) @@ -500,11 +586,11 @@ func TestHandleChange(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - _, err = cc.Checksum(context.TODO(), ref, "d0", true, nil) + _, err = cc.Checksum(context.TODO(), ref, "d0", ChecksumOpts{FollowLinks: true}, nil) require.Error(t, err) require.Equal(t, true, errors.Is(err, errNotFound)) - _, err = cc.Checksum(context.TODO(), ref, "d0/abc", true, nil) + _, err = cc.Checksum(context.TODO(), ref, "d0/abc", ChecksumOpts{FollowLinks: true}, nil) require.Error(t, err) require.Equal(t, true, errors.Is(err, errNotFound)) @@ -541,7 +627,7 @@ func TestHandleRecursiveDir(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - dgst, err := cc.Checksum(context.TODO(), ref, "d0/foo/bar", true, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "d0/foo/bar", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) ch = []string{ @@ -553,11 +639,11 @@ func TestHandleRecursiveDir(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - dgst2, err := cc.Checksum(context.TODO(), ref, "d1", true, nil) + dgst2, err := cc.Checksum(context.TODO(), ref, "d1", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgst2, dgst) - _, err = cc.Checksum(context.TODO(), ref, "", true, nil) + _, err = cc.Checksum(context.TODO(), ref, "", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) } @@ -588,7 +674,7 @@ func TestChecksumUnorderedFiles(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - dgst, err := cc.Checksum(context.TODO(), ref, "d0", true, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "d0", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgst, digest.Digest("sha256:14276c302c940a80f82ca5477bf766c98a24702d6a9948ee71bb277cdad3ae05")) @@ -608,7 +694,7 @@ func TestChecksumUnorderedFiles(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - dgst2, err := cc.Checksum(context.TODO(), ref, "d0", true, nil) + dgst2, err := cc.Checksum(context.TODO(), ref, "d0", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.NotEqual(t, dgst, dgst2) @@ -633,11 +719,11 @@ func TestSymlinkInPathScan(t *testing.T) { } ref := createRef(t, cm, ch) - dgst, err := Checksum(context.TODO(), ref, "d0/def/foo", true, nil) + dgst, err := Checksum(context.TODO(), ref, "d0/def/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = Checksum(context.TODO(), ref, "d0/def/foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "d0/def/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -667,10 +753,10 @@ func TestSymlinkNeedsScan(t *testing.T) { ref := createRef(t, cm, ch) // scan the d0 path containing the symlink that doesn't get followed - _, err = Checksum(context.TODO(), ref, "d0/d1", true, nil) + _, err = Checksum(context.TODO(), ref, "d0/d1", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) - dgst, err := Checksum(context.TODO(), ref, "d0/d1/def/foo", true, nil) + dgst, err := Checksum(context.TODO(), ref, "d0/d1/def/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -697,7 +783,7 @@ func TestSymlinkAbsDirSuffix(t *testing.T) { } ref := createRef(t, cm, ch) - dgst, err := Checksum(context.TODO(), ref, "link/foo", true, nil) + dgst, err := Checksum(context.TODO(), ref, "link/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -732,27 +818,27 @@ func TestSymlinkThroughParent(t *testing.T) { } ref := createRef(t, cm, ch) - dgst, err := Checksum(context.TODO(), ref, "link1/sub/foo", true, nil) + dgst, err := Checksum(context.TODO(), ref, "link1/sub/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = Checksum(context.TODO(), ref, "link2/sub/foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "link2/sub/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = Checksum(context.TODO(), ref, "link3/sub/foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "link3/sub/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = Checksum(context.TODO(), ref, "link4/sub/foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "link4/sub/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = Checksum(context.TODO(), ref, "link5/sub/foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "link5/sub/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = Checksum(context.TODO(), ref, "link1/sub/link/sub/foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "link1/sub/link/sub/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -795,27 +881,27 @@ func TestSymlinkInPathHandleChange(t *testing.T) { err = emit(cc.HandleChange, changeStream(ch)) require.NoError(t, err) - dgst, err := cc.Checksum(context.TODO(), ref, "d1/def/foo", true, nil) + dgst, err := cc.Checksum(context.TODO(), ref, "d1/def/foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgst, err = cc.Checksum(context.TODO(), ref, "d1/def/bar/abc", true, nil) + dgst, err = cc.Checksum(context.TODO(), ref, "d1/def/bar/abc", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) - dgstFileData0, err := cc.Checksum(context.TODO(), ref, "sub/d0", true, nil) + dgstFileData0, err := cc.Checksum(context.TODO(), ref, "sub/d0", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgstDirD0) - dgstFileData0, err = cc.Checksum(context.TODO(), ref, "d1/def/baz", true, nil) + dgstFileData0, err = cc.Checksum(context.TODO(), ref, "d1/def/baz", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgstDirD0) - dgstFileData0, err = cc.Checksum(context.TODO(), ref, "d1/def/bay", true, nil) + dgstFileData0, err = cc.Checksum(context.TODO(), ref, "d1/def/bay", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgstDirD0) - dgstFileData0, err = cc.Checksum(context.TODO(), ref, "link", true, nil) + dgstFileData0, err = cc.Checksum(context.TODO(), ref, "link", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgstDirD0) @@ -846,7 +932,7 @@ func TestPersistence(t *testing.T) { ref := createRef(t, cm, ch) id := ref.ID() - dgst, err := Checksum(context.TODO(), ref, "foo", true, nil) + dgst, err := Checksum(context.TODO(), ref, "foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -856,7 +942,7 @@ func TestPersistence(t *testing.T) { ref, err = cm.Get(context.TODO(), id) require.NoError(t, err) - dgst, err = Checksum(context.TODO(), ref, "foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) @@ -876,7 +962,7 @@ func TestPersistence(t *testing.T) { ref, err = cm.Get(context.TODO(), id) require.NoError(t, err) - dgst, err = Checksum(context.TODO(), ref, "foo", true, nil) + dgst, err = Checksum(context.TODO(), ref, "foo", ChecksumOpts{FollowLinks: true}, nil) require.NoError(t, err) require.Equal(t, dgstFileData0, dgst) } diff --git a/solver/llbsolver/ops/file.go b/solver/llbsolver/ops/file.go index 554532c1845ff..0cd97a17e60c4 100644 --- a/solver/llbsolver/ops/file.go +++ b/solver/llbsolver/ops/file.go @@ -50,7 +50,7 @@ func NewFileOp(v solver.Vertex, op *pb.Op_File, cm cache.Manager, md *metadata.S } func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*solver.CacheMap, bool, error) { - selectors := map[int]map[llbsolver.Selector]struct{}{} + selectors := map[int][]llbsolver.Selector{} invalidSelectors := map[int]struct{}{} actions := make([][]byte, 0, len(f.op.Actions)) @@ -95,7 +95,7 @@ func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol markInvalid(action.Input) processOwner(p.Owner, selectors) if action.SecondaryInput != -1 && int(action.SecondaryInput) < f.numInputs { - addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard, p.FollowSymlink) + addSelector(selectors, int(action.SecondaryInput), p.Src, p.AllowWildcard, p.FollowSymlink, p.IncludePatterns, p.ExcludePatterns) p.Src = path.Base(p.Src) } dt, err = json.Marshal(p) @@ -139,7 +139,7 @@ func (f *fileOp) CacheMap(ctx context.Context, g session.Group, index int) (*sol continue } dgsts := make([][]byte, 0, len(m)) - for k := range m { + for _, k := range m { dgsts = append(dgsts, []byte(k.Path)) } sort.Slice(dgsts, func(i, j int) bool { @@ -179,21 +179,16 @@ func (f *fileOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu return outResults, nil } -func addSelector(m map[int]map[llbsolver.Selector]struct{}, idx int, sel string, wildcard, followLinks bool) { - mm, ok := m[idx] - if !ok { - mm = map[llbsolver.Selector]struct{}{} - m[idx] = mm +func addSelector(m map[int][]llbsolver.Selector, idx int, sel string, wildcard, followLinks bool, includePatterns, excludePatterns []string) { + s := llbsolver.Selector{ + Path: sel, + FollowLinks: followLinks, + Wildcard: wildcard && containsWildcards(sel), + IncludePatterns: includePatterns, + ExcludePatterns: excludePatterns, } - s := llbsolver.Selector{Path: sel} - if wildcard && containsWildcards(sel) { - s.Wildcard = true - } - if followLinks { - s.FollowLinks = true - } - mm[s] = struct{}{} + m[idx] = append(m[idx], s) } func containsWildcards(name string) bool { @@ -209,11 +204,11 @@ func containsWildcards(name string) bool { return false } -func dedupeSelectors(m map[llbsolver.Selector]struct{}) []llbsolver.Selector { +func dedupeSelectors(m []llbsolver.Selector) []llbsolver.Selector { paths := make([]string, 0, len(m)) pathsFollow := make([]string, 0, len(m)) - for sel := range m { - if !sel.Wildcard { + for _, sel := range m { + if !sel.HasWildcardOrFilters() { if sel.FollowLinks { pathsFollow = append(pathsFollow, sel.Path) } else { @@ -232,8 +227,8 @@ func dedupeSelectors(m map[llbsolver.Selector]struct{}) []llbsolver.Selector { selectors = append(selectors, llbsolver.Selector{Path: p, FollowLinks: true}) } - for sel := range m { - if sel.Wildcard { + for _, sel := range m { + if sel.HasWildcardOrFilters() { selectors = append(selectors, sel) } } @@ -245,7 +240,7 @@ func dedupeSelectors(m map[llbsolver.Selector]struct{}) []llbsolver.Selector { return selectors } -func processOwner(chopt *pb.ChownOpt, selectors map[int]map[llbsolver.Selector]struct{}) error { +func processOwner(chopt *pb.ChownOpt, selectors map[int][]llbsolver.Selector) error { if chopt == nil { return nil } @@ -254,7 +249,7 @@ func processOwner(chopt *pb.ChownOpt, selectors map[int]map[llbsolver.Selector]s if u.ByName.Input < 0 { return errors.Errorf("invalid user index %d", u.ByName.Input) } - addSelector(selectors, int(u.ByName.Input), "/etc/passwd", false, true) + addSelector(selectors, int(u.ByName.Input), "/etc/passwd", false, true, nil, nil) } } if chopt.Group != nil { @@ -262,7 +257,7 @@ func processOwner(chopt *pb.ChownOpt, selectors map[int]map[llbsolver.Selector]s if u.ByName.Input < 0 { return errors.Errorf("invalid user index %d", u.ByName.Input) } - addSelector(selectors, int(u.ByName.Input), "/etc/group", false, true) + addSelector(selectors, int(u.ByName.Input), "/etc/group", false, true, nil, nil) } } return nil diff --git a/solver/llbsolver/result.go b/solver/llbsolver/result.go index 4c72353c9854b..6dcca69f6f9af 100644 --- a/solver/llbsolver/result.go +++ b/solver/llbsolver/result.go @@ -16,9 +16,15 @@ import ( ) type Selector struct { - Path string - Wildcard bool - FollowLinks bool + Path string + Wildcard bool + FollowLinks bool + IncludePatterns []string + ExcludePatterns []string +} + +func (sel Selector) HasWildcardOrFilters() bool { + return sel.Wildcard || len(sel.IncludePatterns) != 0 || len(sel.ExcludePatterns) != 0 } func UnlazyResultFunc(ctx context.Context, res solver.Result, g session.Group) error { @@ -50,19 +56,20 @@ func NewContentHashFunc(selectors []Selector) solver.ResultBasedCacheFunc { for i, sel := range selectors { i, sel := i, sel eg.Go(func() error { - if !sel.Wildcard { - dgst, err := contenthash.Checksum(ctx, ref.ImmutableRef, path.Join("/", sel.Path), sel.FollowLinks, s) - if err != nil { - return err - } - dgsts[i] = []byte(dgst) - } else { - dgst, err := contenthash.ChecksumWildcard(ctx, ref.ImmutableRef, path.Join("/", sel.Path), sel.FollowLinks, s) - if err != nil { - return err - } - dgsts[i] = []byte(dgst) + dgst, err := contenthash.Checksum( + ctx, ref.ImmutableRef, path.Join("/", sel.Path), + contenthash.ChecksumOpts{ + Wildcard: sel.Wildcard, + FollowLinks: sel.FollowLinks, + IncludePatterns: sel.IncludePatterns, + ExcludePatterns: sel.ExcludePatterns, + }, + s, + ) + if err != nil { + return err } + dgsts[i] = []byte(dgst) return nil }) }