Merge pull request #4561 from leandrosansilva/feature/exclude-pattern…

…-on-dockerfile

Dockerfile: Allow files to be excluded from cache on COPY and ADD commands

client/llb/fileop.go

@@ -398,6 +398,18 @@ func WithAllowWildcard(b bool) RmOption {
 	})
 }
 
+type excludeOnCopyAction struct {
+	patterns []string
+}
+
+func (e *excludeOnCopyAction) SetCopyOption(i *CopyInfo) {
+	i.ExcludePatterns = append(i.ExcludePatterns, e.patterns...)
+}
+
+func WithExcludePatterns(patterns []string) CopyOption {
+	return &excludeOnCopyAction{patterns}
+}
+
 type fileActionRm struct {
 	file string
 	info RmInfo

docker-bake.hcl

@@ -144,7 +144,7 @@ target "lint" {
   matrix = {
     buildtags = [
       { name = "default", tags = "", target = "golangci-lint" },
-      { name = "labs", tags = "dfrunsecurity dfparents", target = "golangci-lint" },
+      { name = "labs", tags = "dfrunsecurity dfparents dfexcludepatterns", target = "golangci-lint" },
       { name = "nydus", tags = "nydus", target = "golangci-lint" },
       { name = "yaml", tags = "", target = "yamllint" },
       { name = "proto", tags = "", target = "protolint" },

frontend/dockerfile/dockerfile2llb/convert.go

@@ -744,17 +744,18 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 		}
 		if err == nil {
 			err = dispatchCopy(d, copyConfig{
-				params:       c.SourcesAndDest,
-				source:       opt.buildContext,
-				isAddCommand: true,
-				cmdToPrint:   c,
-				chown:        c.Chown,
-				chmod:        c.Chmod,
-				link:         c.Link,
-				keepGitDir:   c.KeepGitDir,
-				checksum:     checksum,
-				location:     c.Location(),
-				opt:          opt,
+				params:          c.SourcesAndDest,
+				excludePatterns: c.ExcludePatterns,
+				source:          opt.buildContext,
+				isAddCommand:    true,
+				cmdToPrint:      c,
+				chown:           c.Chown,
+				chmod:           c.Chmod,
+				link:            c.Link,
+				keepGitDir:      c.KeepGitDir,
+				checksum:        checksum,
+				location:        c.Location(),
+				opt:             opt,
 			})
 		}
 		if err == nil {
@@ -796,16 +797,17 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 			l = src.state
 		}
 		err = dispatchCopy(d, copyConfig{
-			params:       c.SourcesAndDest,
-			source:       l,
-			isAddCommand: false,
-			cmdToPrint:   c,
-			chown:        c.Chown,
-			chmod:        c.Chmod,
-			link:         c.Link,
-			parents:      c.Parents,
-			location:     c.Location(),
-			opt:          opt,
+			params:          c.SourcesAndDest,
+			excludePatterns: c.ExcludePatterns,
+			source:          l,
+			isAddCommand:    false,
+			cmdToPrint:      c,
+			chown:           c.Chown,
+			chmod:           c.Chmod,
+			link:            c.Link,
+			parents:         c.Parents,
+			location:        c.Location(),
+			opt:             opt,
 		})
 		if err == nil {
 			if len(cmd.sources) == 0 {
@@ -1128,6 +1130,13 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 		copyOpt = append(copyOpt, llb.WithUser(cfg.chown))
 	}
 
+	if len(cfg.excludePatterns) > 0 {
+		// in theory we don't need to check whether there are any exclude patterns,
+		// as an empty list is a no-op. However, performing the check makes
+		// the code easier to understand and costs virtually nothing.
+		copyOpt = append(copyOpt, llb.WithExcludePatterns(cfg.excludePatterns))
+	}
+
 	var mode *os.FileMode
 	if cfg.chmod != "" {
 		p, err := strconv.ParseUint(cfg.chmod, 8, 32)
@@ -1331,18 +1340,19 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 }
 
 type copyConfig struct {
-	params       instructions.SourcesAndDest
-	source       llb.State
-	isAddCommand bool
-	cmdToPrint   fmt.Stringer
-	chown        string
-	chmod        string
-	link         bool
-	keepGitDir   bool
-	checksum     digest.Digest
-	parents      bool
-	location     []parser.Range
-	opt          dispatchOpt
+	params          instructions.SourcesAndDest
+	excludePatterns []string
+	source          llb.State
+	isAddCommand    bool
+	cmdToPrint      fmt.Stringer
+	chown           string
+	chmod           string
+	link            bool
+	keepGitDir      bool
+	checksum        digest.Digest
+	parents         bool
+	location        []parser.Range
+	opt             dispatchOpt
 }
 
 func dispatchMaintainer(d *dispatchState, c *instructions.MaintainerCommand) error {

frontend/dockerfile/dockerfile2llb/exclude_patterns_test.go

@@ -0,0 +1,27 @@
+//go:build dfexcludepatterns
+// +build dfexcludepatterns
+
+package dockerfile2llb
+
+import (
+	"testing"
+
+	"github.com/moby/buildkit/util/appcontext"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDockerfileCopyExcludePatterns(t *testing.T) {
+	df := `FROM scratch
+COPY --exclude=src/*.go --exclude=tmp/*.txt dir /sub/
+`
+	_, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
+	assert.NoError(t, err)
+}
+
+func TestDockerfileAddExcludePatterns(t *testing.T) {
+	df := `FROM scratch
+ADD --exclude=src/*.go --exclude=tmp/*.txt dir /sub/
+`
+	_, _, _, err := Dockerfile2LLB(appcontext.Context(), []byte(df), ConvertOpt{})
+	assert.NoError(t, err)
+}

frontend/dockerfile/dockerfile_test.go

@@ -232,6 +232,7 @@ func init() {
 
 func TestIntegration(t *testing.T) {
 	integration.Run(t, allTests, opts...)
+
 	integration.Run(t, securityTests, append(append(opts, securityOpts...),
 		integration.WithMatrix("security.insecure", map[string]interface{}{
 			"granted": securityInsecureGranted,

frontend/dockerfile/docs/reference.md

@@ -1138,8 +1138,8 @@ RUN apt-get update && apt-get install -y ...
 ADD has two forms:
 
 ```dockerfile
-ADD [--chown=<user>:<group>] [--chmod=<perms>] [--checksum=<checksum>] <src>... <dest>
-ADD [--chown=<user>:<group>] [--chmod=<perms>] ["<src>",... "<dest>"]
+ADD [--chown=<user>:<group>] [--chmod=<perms> [--exclude=<exclude>]... [--checksum=<checksum>] <src>... <dest>
+ADD [--chown=<user>:<group>] [--chmod=<perms> [--exclude=<exclude>]... ["<src>",... "<dest>"]
 ```
 
 The latter form is required for paths containing whitespace.
@@ -1158,20 +1158,25 @@ The latter form is required for paths containing whitespace.
 > Only octal notation is currently supported. Non-octal support is tracked in
 > [moby/buildkit#1951](https://github.com/moby/buildkit/issues/1951).
 
+> **Note**
+>
+> The `--exclude` option can be specified multiple times and cause files matching its patterns not to be copied,
+> even if the files paths match the pattern specified in `<src>`. This feature requires the build tag `dfexcludepatterns`.
+
 The `ADD` instruction copies new files, directories or remote file URLs from `<src>`
 and adds them to the filesystem of the image at the path `<dest>`.
 
 Multiple `<src>` resources may be specified but if they are files or
 directories, their paths are interpreted as relative to the source of
 the context of the build.
 
-Each `<src>` may contain wildcards and matching will be done using Go's
+Each `<src>` and `<exclude>` may contain wildcards and matching will be done using Go's
 [filepath.Match](https://golang.org/pkg/path/filepath#Match) rules. For example:
 
-To add all files starting with "hom":
+To add all files starting with "hom", excluding all files with `.txt` and `.md` extensions:
 
 ```dockerfile
-ADD hom* /mydir/
+ADD --exclude=*.txt --exclude=*.md hom* /mydir/
 ```
 
 In the example below, `?` is replaced with any single character, e.g., "home.txt".
@@ -1366,8 +1371,8 @@ See [`COPY --link`](#copy---link).
 COPY has two forms:
 
 ```dockerfile
-COPY [--chown=<user>:<group>] [--chmod=<perms>] <src>... <dest>
-COPY [--chown=<user>:<group>] [--chmod=<perms>] ["<src>",... "<dest>"]
+COPY [--chown=<user>:<group>] [--chmod=<perms>] [--exclude=<exclude>]... <src>... <dest>
+COPY [--chown=<user>:<group>] [--chmod=<perms>] [--exclude=<exclude>]... ["<src>",... "<dest>"]
 ```
 
 This latter form is required for paths containing whitespace
@@ -1380,6 +1385,11 @@ This latter form is required for paths containing whitespace
 > translating user and group names to IDs restricts this feature to only be viable for
 > Linux OS-based containers.
 
+> **Note**
+>
+> The `--exclude` option can be specified multiple times and cause files matching its patterns not to be copied,
+> even if the files paths match the pattern specified in `<src>`. This feature requires the build tag `dfexcludepatterns`.
+
 The `COPY` instruction copies new files or directories from `<src>`
 and adds them to the filesystem of the container at the path `<dest>`.
 
@@ -1390,10 +1400,10 @@ of the build.
 Each `<src>` may contain wildcards and matching will be done using Go's
 [filepath.Match](https://golang.org/pkg/path/filepath#Match) rules. For example:
 
-To add all files starting with "hom":
+To add all files starting with "hom", excluding all files with `.txt` and `.md` extensions:
 
 ```dockerfile
-COPY hom* /mydir/
+COPY --exclude=*.txt --exclude=*.md hom* /mydir/
 ```
 
 In the example below, `?` is replaced with any single character, e.g., "home.txt".