Merge pull request #809 from tonistiigi/fileop

llb: fileop implementation

cache/contenthash/checksum.go

@@ -47,6 +47,10 @@ func Checksum(ctx context.Context, ref cache.ImmutableRef, path string, followLi
 	return getDefaultManager().Checksum(ctx, ref, path, followLinks)
 }
 
+func ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, path string, followLinks bool) (digest.Digest, error) {
+	return getDefaultManager().ChecksumWildcard(ctx, ref, path, followLinks)
+}
+
 func GetCacheContext(ctx context.Context, md *metadata.StorageItem) (CacheContext, error) {
 	return getDefaultManager().GetCacheContext(ctx, md)
 }
@@ -84,6 +88,14 @@ func (cm *cacheManager) Checksum(ctx context.Context, ref cache.ImmutableRef, p
 	return cc.Checksum(ctx, ref, p, followLinks)
 }
 
+func (cm *cacheManager) ChecksumWildcard(ctx context.Context, ref cache.ImmutableRef, p string, followLinks bool) (digest.Digest, error) {
+	cc, err := cm.GetCacheContext(ctx, ensureOriginMetadata(ref.Metadata()))
+	if err != nil {
+		return "", nil
+	}
+	return cc.ChecksumWildcard(ctx, ref, p, followLinks)
+}
+
 func (cm *cacheManager) GetCacheContext(ctx context.Context, md *metadata.StorageItem) (CacheContext, error) {
 	cm.locker.Lock(md.ID())
 	cm.lruMu.Lock()
@@ -343,6 +355,9 @@ func (cc *cacheContext) ChecksumWildcard(ctx context.Context, mountable cache.Mo
 			}
 		}
 	}
+	if len(wildcards) == 0 {
+		return digest.FromBytes([]byte{}), nil
+	}
 
 	if len(wildcards) > 1 {
 		digester := digest.Canonical.Digester()
@@ -543,12 +558,13 @@ func (cc *cacheContext) lazyChecksum(ctx context.Context, m *mount, p string) (*
 }
 
 func (cc *cacheContext) checksum(ctx context.Context, root *iradix.Node, txn *iradix.Txn, m *mount, k []byte, follow bool) (*CacheRecord, bool, error) {
+	origk := k
 	k, cr, err := getFollowLinks(root, k, follow)
 	if err != nil {
 		return nil, false, err
 	}
 	if cr == nil {
-		return nil, false, errors.Wrapf(errNotFound, "%s not found", convertKeyToPath(k))
+		return nil, false, errors.Wrapf(errNotFound, "%q not found", convertKeyToPath(origk))
 	}
 	if cr.Digest != "" {
 		return cr, false, nil

cache/refs.go

@@ -311,7 +311,7 @@ func (sr *mutableRef) updateLastUsed() bool {
 
 func (sr *mutableRef) commit(ctx context.Context) (ImmutableRef, error) {
 	if !sr.mutable || len(sr.refs) == 0 {
-		return nil, errors.Wrapf(errInvalid, "invalid mutable ref")
+		return nil, errors.Wrapf(errInvalid, "invalid mutable ref %p", sr)
 	}
 
 	id := identity.NewID()

client/client_test.go

@@ -51,6 +51,8 @@ func (nopWriteCloser) Close() error { return nil }
 func TestClientIntegration(t *testing.T) {
 	integration.Run(t, []integration.Test{
 		testRelativeWorkDir,
+		testFileOpMkdirMkfile,
+		testFileOpCopyRm,
 		testCallDiskUsage,
 		testBuildMultiMount,
 		testBuildHTTPSource,
@@ -653,6 +655,111 @@ func testRelativeWorkDir(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, []byte("/test1/test2\n"), dt)
 }
 
+func testFileOpMkdirMkfile(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	st := llb.Scratch().
+		File(llb.Mkdir("/foo", 0700).Mkfile("bar", 0600, []byte("contents")))
+
+	def, err := st.Marshal()
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	fi, err := os.Stat(filepath.Join(destDir, "foo"))
+	require.NoError(t, err)
+	require.Equal(t, true, fi.IsDir())
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "bar"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("contents"), dt)
+}
+
+func testFileOpCopyRm(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dir, err := tmpdir(
+		fstest.CreateFile("myfile", []byte("data0"), 0600),
+		fstest.CreateDir("sub", 0700),
+		fstest.CreateFile("sub/foo", []byte("foo0"), 0600),
+		fstest.CreateFile("sub/bar", []byte("bar0"), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	dir2, err := tmpdir(
+		fstest.CreateFile("file2", []byte("file2"), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	st := llb.Scratch().
+		File(
+			llb.Copy(llb.Local("mylocal"), "myfile", "myfile2").
+				Copy(llb.Local("mylocal"), "sub", "out").
+				Rm("out/foo").
+				Copy(llb.Local("mylocal2"), "file2", "/"))
+
+	def, err := st.Marshal()
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			"mylocal":  dir,
+			"mylocal2": dir2,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "myfile2"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("data0"), dt)
+
+	fi, err := os.Stat(filepath.Join(destDir, "out"))
+	require.NoError(t, err)
+	require.Equal(t, true, fi.IsDir())
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "out/bar"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("bar0"), dt)
+
+	_, err = os.Stat(filepath.Join(destDir, "out/foo"))
+	require.Equal(t, true, os.IsNotExist(err))
+
+	dt, err = ioutil.ReadFile(filepath.Join(destDir, "file2"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("file2"), dt)
+
+}
+
 func testCallDiskUsage(t *testing.T, sb integration.Sandbox) {
 	c, err := New(context.TODO(), sb.Address())
 	require.NoError(t, err)