Add IncludePatterns and ExcludePatterns options for Copy

Allow include and exclude patterns to be specified for the "copy" op, similarly to "local". Depends on tonistiigi/fsutil#101 Signed-off-by: Aaron Lehmann <[email protected]>
aaronlehmann · Apr 26, 2021 · 2bab787 · 2bab787
1 parent 75f3315
commit 2bab787
Show file tree

Hide file tree

Showing 18 changed files with 1,363 additions and 281 deletions.
diff --git a/client/client_test.go b/client/client_test.go
@@ -73,6 +73,7 @@ func TestIntegration(t *testing.T) {
 		testRelativeWorkDir,
 		testFileOpMkdirMkfile,
 		testFileOpCopyRm,
+		testFileOpCopyIncludeExclude,
 		testFileOpRmWildcard,
 		testCallDiskUsage,
 		testBuildMultiMount,
@@ -1132,6 +1133,60 @@ func testFileOpCopyRm(t *testing.T, sb integration.Sandbox) {
 	require.Equal(t, []byte("file2"), dt)
 }
 
+func testFileOpCopyIncludeExclude(t *testing.T, sb integration.Sandbox) {
+	requiresLinux(t)
+	c, err := New(context.TODO(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	dir, err := tmpdir(
+		fstest.CreateFile("myfile", []byte("data0"), 0600),
+		fstest.CreateDir("sub", 0700),
+		fstest.CreateFile("sub/foo", []byte("foo0"), 0600),
+		fstest.CreateFile("sub/bar", []byte("bar0"), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	st := llb.Scratch().File(
+		llb.Copy(
+			llb.Local("mylocal"), "/", "/", &llb.CopyInfo{
+				IncludePatterns: []string{"sub/*"},
+				ExcludePatterns: []string{"sub/bar"},
+			},
+		),
+	)
+
+	def, err := st.Marshal(context.TODO())
+	require.NoError(t, err)
+
+	destDir, err := ioutil.TempDir("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(destDir)
+
+	_, err = c.Solve(context.TODO(), def, SolveOpt{
+		Exports: []ExportEntry{
+			{
+				Type:      ExporterLocal,
+				OutputDir: destDir,
+			},
+		},
+		LocalDirs: map[string]string{
+			"mylocal": dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+
+	dt, err := ioutil.ReadFile(filepath.Join(destDir, "sub", "foo"))
+	require.NoError(t, err)
+	require.Equal(t, []byte("foo0"), dt)
+
+	for _, name := range []string{"myfile", "sub/bar"} {
+		_, err = os.Stat(filepath.Join(destDir, name))
+		require.Equal(t, true, errors.Is(err, os.ErrNotExist))
+	}
+}
+
 // testFileOpInputSwap is a regression test that cache is invalidated when subset of fileop is built
 func testFileOpInputSwap(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)

diff --git a/client/llb/fileop.go b/client/llb/fileop.go
@@ -427,6 +427,8 @@ type CopyInfo struct {
 	Mode                *os.FileMode
 	FollowSymlinks      bool
 	CopyDirContentsOnly bool
+	IncludePatterns     []string
+	ExcludePatterns     []string
 	AttemptUnpack       bool
 	CreateDestPath      bool
 	AllowWildcard       bool
@@ -458,6 +460,8 @@ func (a *fileActionCopy) toProtoAction(ctx context.Context, parent string, base
 		Src:                              src,
 		Dest:                             normalizePath(parent, a.dest, true),
 		Owner:                            a.info.ChownOpt.marshal(base),
+		IncludePatterns:                  a.info.IncludePatterns,
+		ExcludePatterns:                  a.info.ExcludePatterns,
 		AllowWildcard:                    a.info.AllowWildcard,
 		AllowEmptyWildcard:               a.info.AllowEmptyWildcard,
 		FollowSymlink:                    a.info.FollowSymlinks,

diff --git a/go.mod b/go.mod
@@ -61,7 +61,7 @@ require (
 	go.etcd.io/bbolt v1.3.5
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
 	golang.org/x/net v0.0.0-20210226172049-e18ecbb05110
-	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210324051608-47abb6519492
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
 	// genproto: the actual version is replaced in replace()
@@ -75,6 +75,8 @@ replace (
 	github.com/golang/protobuf => github.com/golang/protobuf v1.3.5
 	github.com/hashicorp/go-immutable-radix => github.com/tonistiigi/go-immutable-radix v0.0.0-20170803185627-826af9ccf0fe
 	github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305
+	// temporary until https://github.com/tonistiigi/fsutil/pull/101 is merged
+	github.com/tonistiigi/fsutil => github.com/aaronlehmann/fsutil v0.0.0-20210421230417-620ac87d5620
 	// genproto: corresponds to containerd
 	google.golang.org/genproto => google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63
 	// grpc: corresponds to protobuf

diff --git a/go.sum b/go.sum
diff --git a/solver/llbsolver/file/backend.go b/solver/llbsolver/file/backend.go
@@ -187,6 +187,8 @@ func docopy(ctx context.Context, src, dest string, action pb.FileActionCopy, u *
 
 	opt := []copy.Opt{
 		func(ci *copy.CopyInfo) {
+			ci.IncludePatterns = action.IncludePatterns
+			ci.ExcludePatterns = action.ExcludePatterns
 			ci.Chown = ch
 			ci.Utime = timestampToTime(action.Timestamp)
 			if m := int(action.Mode); m != -1 {

diff --git a/solver/pb/ops.pb.go b/solver/pb/ops.pb.go
diff --git a/solver/pb/ops.proto b/solver/pb/ops.proto
@@ -291,6 +291,10 @@ message FileActionCopy {
 	bool allowEmptyWildcard = 10;
 	// optional created time override
 	int64 timestamp = 11;
+	// include only files/dirs matching at least one of these patterns
+	repeated string include_patterns = 12;
+	// exclude files/dir matching any of these patterns (even if they match an include pattern)
+	repeated string exclude_patterns = 13;
 }
 
 message FileActionMkFile {

diff --git a/vendor/github.com/tonistiigi/fsutil/Dockerfile b/vendor/github.com/tonistiigi/fsutil/Dockerfile