dockerfile: implement ADD <git ref>

e.g.,
```dockerfile
FROM alpine
ADD https://github.com/moby/buildkit.git#v0.10.1 /buildkit
```

Close issue 775

Signed-off-by: Akihiro Suda <[email protected]>

frontend/dockerfile/builder/build.go

@@ -27,6 +27,7 @@ import (
 	"github.com/moby/buildkit/solver/errdefs"
 	"github.com/moby/buildkit/solver/pb"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
+	"github.com/moby/buildkit/util/gitutil"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
@@ -70,7 +71,6 @@ const (
 )
 
 var httpPrefix = regexp.MustCompile(`^https?://`)
-var gitURLPathWithFragmentSuffix = regexp.MustCompile(`\.git(?:#.+)?$`)
 
 func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 	opts := c.BuildOpts().Opts
@@ -181,7 +181,11 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 
 	var buildContext *llb.State
 	isNotLocalContext := false
-	if st, ok := detectGitContext(opts[localNameContext], opts[keyContextKeepGitDirArg]); ok {
+	keepGit := false
+	if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil {
+		keepGit = v
+	}
+	if st, ok := detectGitContext(opts[localNameContext], keepGit); ok {
 		if !forceLocalDockerfile {
 			src = *st
 		}
@@ -599,40 +603,21 @@ func filter(opt map[string]string, key string) map[string]string {
 	return m
 }
 
-func detectGitContext(ref, gitContext string) (*llb.State, bool) {
-	found := false
-	if httpPrefix.MatchString(ref) && gitURLPathWithFragmentSuffix.MatchString(ref) {
-		found = true
-	}
-
-	keepGit := false
-	if gitContext != "" {
-		if v, err := strconv.ParseBool(gitContext); err == nil {
-			keepGit = v
-		}
-	}
-
-	for _, prefix := range []string{"git://", "github.com/", "git@"} {
-		if strings.HasPrefix(ref, prefix) {
-			found = true
-			break
-		}
-	}
-	if !found {
+func detectGitContext(ref string, keepGit bool) (*llb.State, bool) {
+	g, err := gitutil.ParseGitRef(ref)
+	if err != nil {
 		return nil, false
 	}
-
-	parts := strings.SplitN(ref, "#", 2)
-	branch := ""
-	if len(parts) > 1 {
-		branch = parts[1]
+	commit := g.Commit
+	if g.SubDir != "" {
+		commit += ":" + g.SubDir
 	}
 	gitOpts := []llb.GitOption{dockerfile2llb.WithInternalName("load git source " + ref)}
 	if keepGit {
 		gitOpts = append(gitOpts, llb.KeepGitDir())
 	}
 
-	st := llb.Git(parts[0], branch, gitOpts...)
+	st := llb.Git(g.Remote, commit, gitOpts...)
 	return &st, true
 }
 
@@ -869,13 +854,13 @@ func contextByName(ctx context.Context, c client.Client, sessionID, name string,
 		}
 		return &st, &img, nil, nil
 	case "git":
-		st, ok := detectGitContext(v, "1")
+		st, ok := detectGitContext(v, true)
 		if !ok {
 			return nil, nil, nil, errors.Errorf("invalid git context %s", v)
 		}
 		return st, nil, nil, nil
 	case "http", "https":
-		st, ok := detectGitContext(v, "1")
+		st, ok := detectGitContext(v, true)
 		if !ok {
 			httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
 			st = &httpst

frontend/dockerfile/dockerfile2llb/convert.go

@@ -26,6 +26,7 @@ import (
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/apicaps"
 	binfotypes "github.com/moby/buildkit/util/buildinfo/types"
+	"github.com/moby/buildkit/util/gitutil"
 	"github.com/moby/buildkit/util/suggest"
 	"github.com/moby/buildkit/util/system"
 	"github.com/moby/sys/signal"
@@ -648,6 +649,7 @@ func dispatch(d *dispatchState, cmd command, opt dispatchOpt) error {
 			chown:        c.Chown,
 			chmod:        c.Chmod,
 			link:         c.Link,
+			keepGitDir:   c.KeepGitDir,
 			location:     c.Location(),
 			opt:          opt,
 		})
@@ -1002,7 +1004,31 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error {
 
 	for _, src := range cfg.params.SourcePaths {
 		commitMessage.WriteString(" " + src)
-		if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
+		gitRef, gitRefErr := gitutil.ParseGitRef(src)
+		if gitRefErr == nil && !gitRef.IndistinguishableFromLocal {
+			if !cfg.isAddCommand {
+				return errors.New("source can't be a git ref for COPY")
+			}
+			// TODO: print a warning (not an error) if gitRef.UnencryptedTCP is true
+			commit := gitRef.Commit
+			if gitRef.SubDir != "" {
+				commit += ":" + gitRef.SubDir
+			}
+			var gitOptions []llb.GitOption
+			if cfg.keepGitDir {
+				gitOptions = append(gitOptions, llb.KeepGitDir())
+			}
+			st := llb.Git(gitRef.Remote, commit, gitOptions...)
+			opts := append([]llb.CopyOption{&llb.CopyInfo{
+				Mode:           mode,
+				CreateDestPath: true,
+			}}, copyOpt...)
+			if a == nil {
+				a = llb.Copy(st, "/", dest, opts...)
+			} else {
+				a = a.Copy(st, "/", dest, opts...)
+			}
+		} else if strings.HasPrefix(src, "http://") || strings.HasPrefix(src, "https://") {
 			if !cfg.isAddCommand {
 				return errors.New("source can't be a URL for COPY")
 			}
@@ -1125,6 +1151,7 @@ type copyConfig struct {
 	chown        string
 	chmod        string
 	link         bool
+	keepGitDir   bool
 	location     []parser.Range
 	opt          dispatchOpt
 }

frontend/dockerfile/dockerfile_test.go

@@ -18,6 +18,7 @@ import (
 	"sort"
 	"strings"
 	"testing"
+	"text/template"
 	"time"
 
 	"github.com/containerd/containerd"
@@ -133,6 +134,7 @@ var allTests = integration.TestFuncs(
 	testCopyVarSubstitution,
 	testCopyWildcards,
 	testCopyRelative,
+	testAddGit,
 	testAddURLChmod,
 	testTarContext,
 	testTarContextExternalDockerfile,
@@ -3466,6 +3468,81 @@ COPY --from=build /dest /dest
 	require.Equal(t, []byte("0644\n0755\n0413\n"), dt)
 }
 
+func testAddGit(t *testing.T, sb integration.Sandbox) {
+	f := getFrontend(t, sb)
+
+	gitDir, err := os.MkdirTemp("", "buildkit")
+	require.NoError(t, err)
+	defer os.RemoveAll(gitDir)
+	gitCommands := []string{
+		"git init",
+		"git config --local user.email test",
+		"git config --local user.name test",
+	}
+	makeCommit := func(tag string) []string {
+		return []string{
+			"echo foo of " + tag + " >foo",
+			"git add foo",
+			"git commit -m " + tag,
+			"git tag " + tag,
+		}
+	}
+	gitCommands = append(gitCommands, makeCommit("v0.0.1")...)
+	gitCommands = append(gitCommands, makeCommit("v0.0.2")...)
+	gitCommands = append(gitCommands, makeCommit("v0.0.3")...)
+	gitCommands = append(gitCommands, "git update-server-info")
+	err = runShell(gitDir, gitCommands...)
+	require.NoError(t, err)
+
+	server := httptest.NewServer(http.FileServer(http.Dir(filepath.Join(gitDir))))
+	defer server.Close()
+	serverURL := server.URL
+	t.Logf("serverURL=%q", serverURL)
+
+	dockerfile, err := applyTemplate(`
+FROM alpine
+
+# Basic case
+ADD {{.ServerURL}}/.git#v0.0.1 /x
+RUN cd /x && \
+  [ "$(cat foo)" = "foo of v0.0.1" ]
+
+# Complicated case
+ARG REPO="{{.ServerURL}}/.git"
+ARG TAG="v0.0.2"
+ADD --keep-git-dir=true --chown=4242:8484 ${REPO}#${TAG} /buildkit-chowned
+RUN apk add git
+USER 4242
+RUN cd /buildkit-chowned && \
+  [ "$(cat foo)" = "foo of v0.0.2" ] && \
+  [ "$(stat -c %u foo)" = "4242" ] && \
+  [ "$(stat -c %g foo)" = "8484" ] && \
+  [ -z "$(git status -s)" ]
+`, map[string]string{
+		"ServerURL": serverURL,
+	})
+	require.NoError(t, err)
+	t.Logf("dockerfile=%s", dockerfile)
+
+	dir, err := tmpdir(
+		fstest.CreateFile("Dockerfile", []byte(dockerfile), 0600),
+	)
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	c, err := client.New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	_, err = f.Solve(sb.Context(), c, client.SolveOpt{
+		LocalDirs: map[string]string{
+			builder.DefaultLocalNameDockerfile: dir,
+			builder.DefaultLocalNameContext:    dir,
+		},
+	}, nil)
+	require.NoError(t, err)
+}
+
 func testDockerfileFromGit(t *testing.T, sb integration.Sandbox) {
 	f := getFrontend(t, sb)
 
@@ -6107,3 +6184,15 @@ func readImage(ctx context.Context, p content.Provider, desc ocispecs.Descriptor
 	}
 	return ii, nil
 }
+
+func applyTemplate(tmpl string, x interface{}) (string, error) {
+	var buf bytes.Buffer
+	parsed, err := template.New("").Parse(tmpl)
+	if err != nil {
+		return "", err
+	}
+	if err := parsed.Execute(&buf, x); err != nil {
+		return "", err
+	}
+	return buf.String(), nil
+}

frontend/dockerfile/docs/syntax.md

@@ -25,6 +25,44 @@ incrementing the major component of a version and you may want to pin the image
 change in between releases on labs channel, the old versions are guaranteed to be backward compatible.
 
 
+## Adding a git repository `ADD <git ref> <dir>`
+
+<!-- TODO: dockerfile-upstream:master -> dockerfile:1.5 after the release of 1.5 -->
+To use this instruction, set the frontend to `docker/dockerfile-upstream:master`:
+
+```dockerfile
+# syntax=docker/dockerfile-upstream:master`
+```
+
+This instruction allows you to add a git repository to an image directly, without using the `git` command inside the image.
+
+```dockerfile
+# syntax=docker/dockerfile-upstream:master
+FROM alpine
+ADD --keep-git-dir=true https://github.com/moby/buildkit.git#v0.10.1 /buildkit
+```
+
+The `--keep-git-dir=true` flag adds the `.git` directory. This flag defaults to false.
+
+
+### Adding a private git repository
+To add a private repo via SSH, create a Dockerfile with the following syntax:
+```dockerfile
+# syntax = docker/dockerfile-upstream:master
+FROM alpine
+ADD [email protected]:foo/bar.git /bar
+```
+
+This Dockerfile can be built with `docker build --ssh` or `buildctl build --ssh`, e.g.,
+
+```console
+$ docker build --ssh default
+```
+
+```console
+$ buildctl build --frontend=dockerfile.v0 --local context=. --local dockerfile=. --ssh default
+```
+
 ## Linked copies `COPY --link`, `ADD --link`
 
 To use this flag set Dockerfile version to at least `1.4`.

frontend/dockerfile/instructions/commands.go

@@ -224,9 +224,10 @@ func (s *SourcesAndDest) ExpandRaw(expander SingleWordExpander) error {
 type AddCommand struct {
 	withNameAndCode
 	SourcesAndDest
-	Chown string
-	Chmod string
-	Link  bool
+	Chown      string
+	Chmod      string
+	Link       bool
+	KeepGitDir bool // whether to keep .git dir, only meaningful for git sources
 }
 
 // Expand variables

frontend/dockerfile/instructions/parse.go

@@ -281,6 +281,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 	flChown := req.flags.AddString("chown", "")
 	flChmod := req.flags.AddString("chmod", "")
 	flLink := req.flags.AddBool("link", false)
+	flKeepGitDir := req.flags.AddBool("keep-git-dir", false)
 	if err := req.flags.Parse(); err != nil {
 		return nil, err
 	}
@@ -296,6 +297,7 @@ func parseAdd(req parseRequest) (*AddCommand, error) {
 		Chown:           flChown.Value,
 		Chmod:           flChmod.Value,
 		Link:            flLink.Value == "true",
+		KeepGitDir:      flKeepGitDir.Value == "true",
 	}, nil
 }