dockerfile: add support for named contexts

Stages and implicit stages from image names can be redefined with build options. This enables using more that one source directory and reusing results from other builds. This can also be used to use a local image from other build without including a registry. Contexts need to be defined as `context:name=` frontend options. The value can be image, git repository, URL, local directory or a frontend input. Signed-off-by: Tonis Tiigi <[email protected]>
moby · Dec 13, 2021 · eedec9c · eedec9c
1 parent 8700be3
commit eedec9c
Show file tree

Hide file tree

Showing 5 changed files with 360 additions and 4 deletions.
diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go
@@ -14,6 +14,7 @@ import (
 	"strings"
 
 	"github.com/containerd/containerd/platforms"
+	"github.com/docker/distribution/reference"
 	"github.com/docker/go-units"
 	controlapi "github.com/moby/buildkit/api/services/control"
 	"github.com/moby/buildkit/client/llb"
@@ -449,6 +450,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) {
 					LLBCaps:           &caps,
 					SourceMap:         sourceMap,
 					Hostname:          opts[keyHostname],
+					ContextByName:     contextByName(c, tp),
 				})
 
 				if err != nil {
@@ -767,6 +769,79 @@ func scopeToSubDir(c *llb.State, fileop bool, dir string) *llb.State {
 	return &bc
 }
 
+func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) {
+	return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) {
+		named, err := reference.ParseNormalizedNamed(name)
+		if err != nil {
+			return nil, nil, errors.Wrapf(err, "invalid context name %s", name)
+		}
+		name = strings.TrimSuffix(reference.FamiliarString(named), ":latest")
+
+		opts := c.BuildOpts().Opts
+		v, ok := opts["context:"+name]
+		if !ok {
+			return nil, nil, nil
+		}
+
+		vv := strings.SplitN(v, ":", 2)
+		if len(vv) != 2 {
+			return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name)
+		}
+		switch vv[0] {
+		case "docker-image":
+			st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c))
+			return &st, nil, nil
+		case "git":
+			st, ok := detectGitContext(v, "")
+			if !ok {
+				return nil, nil, errors.Errorf("invalid git context %s", v)
+			}
+			return st, nil, nil
+		case "http", "https":
+			st, ok := detectGitContext(v, "")
+			if !ok {
+				httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v))
+				st = &httpst
+			}
+			return st, nil, nil
+		case "local":
+			st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name))
+			return &st, nil, nil
+		case "input":
+			inputs, err := c.Inputs(ctx)
+			if err != nil {
+				return nil, nil, err
+			}
+			st, ok := inputs[vv[1]]
+			if !ok {
+				return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name)
+			}
+			md, ok := opts["input-metadata:"+vv[1]]
+			if ok {
+				m := make(map[string][]byte)
+				if err := json.Unmarshal([]byte(md), &m); err != nil {
+					return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md)
+				}
+				dt, ok := m["containerimage.config"]
+				if ok {
+					st, err = st.WithImageConfig([]byte(dt))
+					if err != nil {
+						return nil, nil, err
+					}
+					var img dockerfile2llb.Image
+					if err := json.Unmarshal(dt, &img); err != nil {
+						return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name)
+					}
+					return &st, &img, nil
+				}
+			}
+			return &st, nil, nil
+		default:
+			return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name)
+		}
+	}
+}
+
 func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error {
 	if sm == nil {
 		return err

diff --git a/frontend/dockerfile/builder/caps.go b/frontend/dockerfile/builder/caps.go
@@ -12,6 +12,7 @@ import (
 var enabledCaps = map[string]struct{}{
 	"moby.buildkit.frontend.inputs":      {},
 	"moby.buildkit.frontend.subrequests": {},
+	"moby.buildkit.frontend.contexts":    {},
 }
 
 func validateCaps(req string) (forward bool, err error) {

diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile
@@ -29,7 +29,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \
 
 FROM scratch AS release
 LABEL moby.buildkit.frontend.network.none="true"
-LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests"
+LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts"
 COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend
 ENTRYPOINT ["/bin/dockerfile-frontend"]
 

diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go
@@ -68,9 +68,14 @@ type ConvertOpt struct {
 	ContextLocalName  string
 	SourceMap         *llb.SourceMap
 	Hostname          string
+	ContextByName     func(context.Context, string) (*llb.State, *Image, error)
 }
 
 func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) {
+	if opt.ContextByName == nil {
+		opt.ContextByName = func(context.Context, string) (*llb.State, *Image, error) { return nil, nil, nil }
+	}
+
 	if len(dt) == 0 {
 		return nil, nil, errors.Errorf("the Dockerfile cannot be empty")
 	}
@@ -128,13 +133,30 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 		st.BaseName = name
 
 		ds := &dispatchState{
-			stage:          st,
 			deps:           make(map[*dispatchState]struct{}),
 			ctxPaths:       make(map[string]struct{}),
 			stageName:      st.Name,
 			prefixPlatform: opt.PrefixPlatform,
 		}
 
+		if st.Name != "" {
+			s, img, err := opt.ContextByName(ctx, st.Name)
+			if err != nil {
+				return nil, nil, err
+			}
+			if s != nil {
+				ds.noinit = true
+				ds.state = *s
+				if img != nil {
+					ds.image = *img
+				}
+				allDispatchStates.addState(ds)
+				continue
+			}
+		}
+
+		ds.stage = st
+
 		if st.Name == "" {
 			ds.stageName = fmt.Sprintf("stage-%d", i)
 		}
@@ -232,7 +254,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 	for i, d := range allDispatchStates.states {
 		reachable := isReachable(target, d)
 		// resolve image config for every stage
-		if d.base == nil {
+		if d.base == nil && !d.noinit {
 			if d.stage.BaseName == emptyImageName {
 				d.state = llb.Scratch()
 				d.image = emptyImage(platformOpt.targetPlatform)
@@ -255,8 +277,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State,
 						platform = &platformOpt.targetPlatform
 					}
 					d.stage.BaseName = reference.TagNameOnly(ref).String()
+
 					var isScratch bool
-					if metaResolver != nil && reachable {
+					st, img, err := opt.ContextByName(ctx, d.stage.BaseName)
+					if err != nil {
+						return err
+					}
+					if st != nil {
+						if img != nil {
+							d.image = *img
+						} else {
+							d.image = emptyImage(platformOpt.targetPlatform)
+						}
+						d.state = *st
+						d.platform = platform
+						return nil
+					}
+					if reachable {
 						prefix := "["
 						if opt.PrefixPlatform && platform != nil {
 							prefix += platforms.Format(*platform) + " "
@@ -610,6 +647,7 @@ type dispatchState struct {
 	platform       *ocispecs.Platform
 	stage          instructions.Stage
 	base           *dispatchState
+	noinit         bool
 	deps           map[*dispatchState]struct{}
 	buildArgs      []instructions.KeyValuePairOptional
 	commands       []command