Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

llb: fileop implementation #809

Merged
merged 25 commits into from
Mar 18, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
5b4841f
llb: initial fileop implementation
tonistiigi Jan 23, 2019
431d11d
llb: add timestamp override to fileop
tonistiigi Feb 1, 2019
89e6614
solver: change uid to uint
tonistiigi Feb 1, 2019
a443cff
fileop: resolve review comments
tonistiigi Feb 5, 2019
9fb1f09
llbsolver: fileop base
tonistiigi Feb 8, 2019
b2b0e3d
fileop: add release support and tests
tonistiigi Feb 20, 2019
2be999b
fileop: llbsolver implementation
tonistiigi Feb 21, 2019
81a5fa5
llbsolver: fileop implementation
tonistiigi Feb 27, 2019
4ffd797
fileop: connect with contenthash
tonistiigi Mar 3, 2019
171feaa
vendor: add fsutil copy package
tonistiigi Mar 15, 2019
7210bf6
fileop: add chown support
tonistiigi Mar 15, 2019
8a4674b
fileop: add dockerfile support
tonistiigi Mar 8, 2019
0d17ac3
fileop: updates with new fsutil copy pkg
tonistiigi Mar 9, 2019
33955c9
vendor: revendor new packages
tonistiigi Mar 11, 2019
bb11c8c
dockerfile: empty wildcard regression test
tonistiigi Mar 10, 2019
39ba2ed
dockerfile: regression test for workdir creation
tonistiigi Mar 10, 2019
369f748
dockerfile: regression test for wildcard unpack
tonistiigi Mar 10, 2019
c4ef668
dockerfile: regression test for existing dest dir perms
tonistiigi Mar 11, 2019
c57e5b2
dockerfile: add regression test for wildcard cache
tonistiigi Mar 10, 2019
637bec7
dockerfile: make fileop default
tonistiigi Mar 11, 2019
f38d971
dockerfile: add matrix testing for non-fileop versions
tonistiigi Mar 11, 2019
a16b47f
integration: disable gc
tonistiigi Mar 11, 2019
c6149da
fileop: review fixes
tonistiigi Mar 11, 2019
6fd30e7
dockerfile: add more tests to fix fileop
Mar 15, 2019
5effbff
dockerfile: improve symlinks copy tests
tonistiigi Mar 16, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion cache/contenthash/checksum.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ func Checksum(ctx context.Context, ref cache.ImmutableRef, path string, followLi
return getDefaultManager().Checksum(ctx, ref, path, followLinks)
}

func ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, path string, followLinks bool) (digest.Digest, error) {
return getDefaultManager().ChecksumWildcard(ctx, ref, path, followLinks)
}

func GetCacheContext(ctx context.Context, md *metadata.StorageItem) (CacheContext, error) {
return getDefaultManager().GetCacheContext(ctx, md)
}
Expand Down Expand Up @@ -84,6 +88,14 @@ func (cm *cacheManager) Checksum(ctx context.Context, ref cache.ImmutableRef, p
return cc.Checksum(ctx, ref, p, followLinks)
}

func (cm *cacheManager) ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool) (digest.Digest, error) {
cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata()))
if err != nil {
return "", nil
}
return cc.ChecksumWildcard(ctx, ref, p, followLinks)
}

func (cm *cacheManager) GetCacheContext(ctx context.Context, md *metadata.StorageItem) (CacheContext, error) {
cm.locker.Lock(md.ID())
cm.lruMu.Lock()
Expand Down Expand Up @@ -343,6 +355,9 @@ func (cc *cacheContext) ChecksumWildcard(ctx context.Context, mountable cache.Mo
}
}
}
if len(wildcards) == 0 {
return digest.FromBytes([]byte{}), nil
}

if len(wildcards) > 1 {
digester := digest.Canonical.Digester()
Expand Down Expand Up @@ -543,12 +558,13 @@ func (cc *cacheContext) lazyChecksum(ctx context.Context, m *mount, p string) (*
}

func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node, txn *iradix.Txn, m *mount, k []byte, follow bool) (*CacheRecord, bool, error) {
origk := k
k, cr, err := getFollowLinks(root, k, follow)
if err != nil {
return nil, false, err
}
if cr == nil {
return nil, false, errors.Wrapf(errNotFound, "%s not found", convertKeyToPath(k))
return nil, false, errors.Wrapf(errNotFound, "%q not found", convertKeyToPath(origk))
}
if cr.Digest != "" {
return cr, false, nil
Expand Down
2 changes: 1 addition & 1 deletion cache/refs.go
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ func (sr *mutableRef) updateLastUsed() bool {

func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) {
if !sr.mutable || len(sr.refs) == 0 {
return nil, errors.Wrapf(errInvalid, "invalid mutable ref")
return nil, errors.Wrapf(errInvalid, "invalid mutable ref %p", sr)
}

id := identity.NewID()
Expand Down
107 changes: 107 additions & 0 deletions client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ func (nopWriteCloser) Close() error { return nil }
func TestClientIntegration(t *testing.T) {
integration.Run(t, []integration.Test{
testRelativeWorkDir,
testFileOpMkdirMkfile,
testFileOpCopyRm,
testCallDiskUsage,
testBuildMultiMount,
testBuildHTTPSource,
Expand Down Expand Up @@ -653,6 +655,111 @@ func testRelativeWorkDir(t *testing.T, sb integration.Sandbox) {
require.Equal(t, []byte("/test1/test2\n"), dt)
}

func testFileOpMkdirMkfile(t *testing.T, sb integration.Sandbox) {
requiresLinux(t)
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()

st := llb.Scratch().
File(llb.Mkdir("/foo", 0700).Mkfile("bar", 0600, []byte("contents")))

def, err := st.Marshal()
require.NoError(t, err)

destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)

_, err = c.Solve(context.TODO(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
}, nil)
require.NoError(t, err)

fi, err := os.Stat(filepath.Join(destDir, "foo"))
require.NoError(t, err)
require.Equal(t, true, fi.IsDir())

dt, err := ioutil.ReadFile(filepath.Join(destDir, "bar"))
require.NoError(t, err)
require.Equal(t, []byte("contents"), dt)
}

func testFileOpCopyRm(t *testing.T, sb integration.Sandbox) {
requiresLinux(t)
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
defer c.Close()

dir, err := tmpdir(
fstest.CreateFile("myfile", []byte("data0"), 0600),
fstest.CreateDir("sub", 0700),
fstest.CreateFile("sub/foo", []byte("foo0"), 0600),
fstest.CreateFile("sub/bar", []byte("bar0"), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)

dir2, err := tmpdir(
fstest.CreateFile("file2", []byte("file2"), 0600),
)
require.NoError(t, err)
defer os.RemoveAll(dir)

st := llb.Scratch().
File(
llb.Copy(llb.Local("mylocal"), "myfile", "myfile2").
Copy(llb.Local("mylocal"), "sub", "out").
Rm("out/foo").
Copy(llb.Local("mylocal2"), "file2", "/"))

def, err := st.Marshal()
require.NoError(t, err)

destDir, err := ioutil.TempDir("", "buildkit")
require.NoError(t, err)
defer os.RemoveAll(destDir)

_, err = c.Solve(context.TODO(), def, SolveOpt{
Exports: []ExportEntry{
{
Type: ExporterLocal,
OutputDir: destDir,
},
},
LocalDirs: map[string]string{
"mylocal": dir,
"mylocal2": dir2,
},
}, nil)
require.NoError(t, err)

dt, err := ioutil.ReadFile(filepath.Join(destDir, "myfile2"))
require.NoError(t, err)
require.Equal(t, []byte("data0"), dt)

fi, err := os.Stat(filepath.Join(destDir, "out"))
require.NoError(t, err)
require.Equal(t, true, fi.IsDir())

dt, err = ioutil.ReadFile(filepath.Join(destDir, "out/bar"))
require.NoError(t, err)
require.Equal(t, []byte("bar0"), dt)

_, err = os.Stat(filepath.Join(destDir, "out/foo"))
require.Equal(t, true, os.IsNotExist(err))

dt, err = ioutil.ReadFile(filepath.Join(destDir, "file2"))
require.NoError(t, err)
require.Equal(t, []byte("file2"), dt)

}

func testCallDiskUsage(t *testing.T, sb integration.Sandbox) {
c, err := New(context.TODO(), sb.Address())
require.NoError(t, err)
Expand Down
Loading