From eedec9c977dd829c8aafd3fbef2dc1d74c94622a Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 12 Dec 2021 20:39:36 -0800 Subject: [PATCH] dockerfile: add support for named contexts Stages and implicit stages from image names can be redefined with build options. This enables using more that one source directory and reusing results from other builds. This can also be used to use a local image from other build without including a registry. Contexts need to be defined as `context:name=` frontend options. The value can be image, git repository, URL, local directory or a frontend input. Signed-off-by: Tonis Tiigi --- frontend/dockerfile/builder/build.go | 75 ++++++ frontend/dockerfile/builder/caps.go | 1 + .../cmd/dockerfile-frontend/Dockerfile | 2 +- frontend/dockerfile/dockerfile2llb/convert.go | 44 +++- frontend/dockerfile/dockerfile_test.go | 242 ++++++++++++++++++ 5 files changed, 360 insertions(+), 4 deletions(-) diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index 329c87a6b1cde..fb1dd35c2a851 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/containerd/containerd/platforms" + "github.com/docker/distribution/reference" "github.com/docker/go-units" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/llb" @@ -449,6 +450,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { LLBCaps: &caps, SourceMap: sourceMap, Hostname: opts[keyHostname], + ContextByName: contextByName(c, tp), }) if err != nil { @@ -767,6 +769,79 @@ func scopeToSubDir(c *llb.State, fileop bool, dir string) *llb.State { return &bc } +func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { + return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) { + named, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, nil, errors.Wrapf(err, "invalid context name %s", name) + } + name = strings.TrimSuffix(reference.FamiliarString(named), ":latest") + + opts := c.BuildOpts().Opts + v, ok := opts["context:"+name] + if !ok { + return nil, nil, nil + } + + vv := strings.SplitN(v, ":", 2) + if len(vv) != 2 { + return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) + } + switch vv[0] { + case "docker-image": + st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) + return &st, nil, nil + case "git": + st, ok := detectGitContext(v, "") + if !ok { + return nil, nil, errors.Errorf("invalid git context %s", v) + } + return st, nil, nil + case "http", "https": + st, ok := detectGitContext(v, "") + if !ok { + httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) + st = &httpst + } + return st, nil, nil + case "local": + st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) + return &st, nil, nil + case "input": + inputs, err := c.Inputs(ctx) + if err != nil { + return nil, nil, err + } + st, ok := inputs[vv[1]] + if !ok { + return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + } + md, ok := opts["input-metadata:"+vv[1]] + if ok { + m := make(map[string][]byte) + if err := json.Unmarshal([]byte(md), &m); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) + } + dt, ok := m["containerimage.config"] + if ok { + st, err = st.WithImageConfig([]byte(dt)) + if err != nil { + return nil, nil, err + } + var img dockerfile2llb.Image + if err := json.Unmarshal(dt, &img); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) + } + return &st, &img, nil + } + } + return &st, nil, nil + default: + return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) + } + } +} + func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error { if sm == nil { return err diff --git a/frontend/dockerfile/builder/caps.go b/frontend/dockerfile/builder/caps.go index 279701154eac2..3c78cd56c4da0 100644 --- a/frontend/dockerfile/builder/caps.go +++ b/frontend/dockerfile/builder/caps.go @@ -12,6 +12,7 @@ import ( var enabledCaps = map[string]struct{}{ "moby.buildkit.frontend.inputs": {}, "moby.buildkit.frontend.subrequests": {}, + "moby.buildkit.frontend.contexts": {}, } func validateCaps(req string) (forward bool, err error) { diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index b8f4e0e857dff..f97d44315d74c 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -29,7 +29,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \ FROM scratch AS release LABEL moby.buildkit.frontend.network.none="true" -LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests" +LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts" COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend ENTRYPOINT ["/bin/dockerfile-frontend"] diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 0a95a2f8a5be0..ca8a024aa47dc 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -68,9 +68,14 @@ type ConvertOpt struct { ContextLocalName string SourceMap *llb.SourceMap Hostname string + ContextByName func(context.Context, string) (*llb.State, *Image, error) } func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) { + if opt.ContextByName == nil { + opt.ContextByName = func(context.Context, string) (*llb.State, *Image, error) { return nil, nil, nil } + } + if len(dt) == 0 { return nil, nil, errors.Errorf("the Dockerfile cannot be empty") } @@ -128,13 +133,30 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, st.BaseName = name ds := &dispatchState{ - stage: st, deps: make(map[*dispatchState]struct{}), ctxPaths: make(map[string]struct{}), stageName: st.Name, prefixPlatform: opt.PrefixPlatform, } + if st.Name != "" { + s, img, err := opt.ContextByName(ctx, st.Name) + if err != nil { + return nil, nil, err + } + if s != nil { + ds.noinit = true + ds.state = *s + if img != nil { + ds.image = *img + } + allDispatchStates.addState(ds) + continue + } + } + + ds.stage = st + if st.Name == "" { ds.stageName = fmt.Sprintf("stage-%d", i) } @@ -232,7 +254,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, for i, d := range allDispatchStates.states { reachable := isReachable(target, d) // resolve image config for every stage - if d.base == nil { + if d.base == nil && !d.noinit { if d.stage.BaseName == emptyImageName { d.state = llb.Scratch() d.image = emptyImage(platformOpt.targetPlatform) @@ -255,8 +277,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, platform = &platformOpt.targetPlatform } d.stage.BaseName = reference.TagNameOnly(ref).String() + var isScratch bool - if metaResolver != nil && reachable { + st, img, err := opt.ContextByName(ctx, d.stage.BaseName) + if err != nil { + return err + } + if st != nil { + if img != nil { + d.image = *img + } else { + d.image = emptyImage(platformOpt.targetPlatform) + } + d.state = *st + d.platform = platform + return nil + } + if reachable { prefix := "[" if opt.PrefixPlatform && platform != nil { prefix += platforms.Format(*platform) + " " @@ -610,6 +647,7 @@ type dispatchState struct { platform *ocispecs.Platform stage instructions.Stage base *dispatchState + noinit bool deps map[*dispatchState]struct{} buildArgs []instructions.KeyValuePairOptional commands []command diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 66856008f1c21..fa67fa5e7df63 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -39,6 +39,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/upload/uploadprovider" "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/testutil" "github.com/moby/buildkit/util/testutil/httpserver" @@ -118,6 +119,9 @@ var allTests = integration.TestFuncs( testShmSize, testUlimit, testCgroupParent, + testNamedImageContext, + testNamedLocalContext, + testNamedInputContext, ) var fileOpTests = integration.TestFuncs( @@ -159,6 +163,7 @@ var securityOpts []integration.TestOpt type frontend interface { Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error) + SolveGateway(context.Context, gateway.Client, gateway.SolveRequest) (*gateway.Result, error) DFCmdArgs(string, string) (string, string) RequiresBuildctl(t *testing.T) } @@ -5406,6 +5411,222 @@ COPY --from=base /out / require.Contains(t, strings.TrimSpace(string(dt)), `/foocgroup/buildkit/`) } +func testNamedImageContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:busybox": "docker-image://alpine", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedLocalContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + outf := []byte(`dummy-result`) + + dir2, err := tmpdir( + fstest.CreateFile("out", outf, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir2) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:base": "local:basedir", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + "basedir": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedInputContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM alpine +ENV FOO=bar +RUN echo first > /out +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dockerfile2 := []byte(` +FROM base AS build +RUN echo "foo is $FOO" > /foo +FROM scratch +COPY --from=build /foo /out / +`) + + dir2, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile2, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{}) + if err != nil { + return nil, err + } + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + st, err := ref.ToState() + if err != nil { + return nil, err + } + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + dt, ok := res.Metadata["containerimage.config"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + + res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: map[string]string{ + "dockerfilekey": builder.DefaultLocalNameDockerfile + "2", + "context:base": "input:base", + "input-metadata:base": string(dt), + }, + FrontendInputs: map[string]*pb.Definition{ + "base": def.ToPB(), + }, + }) + if err != nil { + return nil, err + } + return res, nil + } + + product := "buildkit_test" + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Build(ctx, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + builder.DefaultLocalNameDockerfile + "2": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, product, b, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.Equal(t, "first\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar\n", string(dt)) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil { @@ -5542,6 +5763,11 @@ func (f *builtinFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *builtinFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "dockerfile.v0" + return c.Solve(ctx, req) +} + func (f *builtinFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend dockerfile.v0") } @@ -5556,6 +5782,13 @@ func (f *clientFrontend) Solve(ctx context.Context, c *client.Client, opt client return c.Build(ctx, opt, "", builder.Build, statusChan) } +func (f *clientFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + if req.Frontend == "" && req.Definition == nil { + req.Frontend = "dockerfile.v0" + } + return c.Solve(ctx, req) +} + func (f *clientFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return "", "" } @@ -5578,6 +5811,15 @@ func (f *gatewayFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *gatewayFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "gateway.v0" + if req.FrontendOpt == nil { + req.FrontendOpt = make(map[string]string) + } + req.FrontendOpt["source"] = f.gw + return c.Solve(ctx, req) +} + func (f *gatewayFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend gateway.v0 --opt=source="+f.gw) }