From 0a037f8c43a6082b7e2cfdea79782e6634811f72 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 15 Oct 2021 10:37:50 +0200 Subject: [PATCH] Add support for classic builder Signed-off-by: Ulysses Souza --- go.sum | 1 + pkg/compose/build.go | 60 ++------ pkg/compose/build_buildkit.go | 72 +++++++++ pkg/compose/build_classic.go | 273 ++++++++++++++++++++++++++++++++++ pkg/compose/build_win.go | 28 ---- 5 files changed, 357 insertions(+), 77 deletions(-) create mode 100644 pkg/compose/build_buildkit.go create mode 100644 pkg/compose/build_classic.go delete mode 100644 pkg/compose/build_win.go diff --git a/go.sum b/go.sum index 6d523aaf814..3ba5d55b8d1 100644 --- a/go.sum +++ b/go.sum @@ -771,6 +771,7 @@ github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+S github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/symlink v0.1.0 h1:MTFZ74KtNI6qQQpuBxU+uKCim4WtOMokr03hCfJcazE= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= diff --git a/pkg/compose/build.go b/pkg/compose/build.go index a1ca27669ce..fcbe5382600 100644 --- a/pkg/compose/build.go +++ b/pkg/compose/build.go @@ -25,10 +25,12 @@ import ( "github.com/compose-spec/compose-go/types" "github.com/containerd/containerd/platforms" "github.com/docker/buildx/build" - "github.com/docker/buildx/driver" _ "github.com/docker/buildx/driver/docker" // required to get default driver registered "github.com/docker/buildx/util/buildflags" xprogress "github.com/docker/buildx/util/progress" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/flags" + "github.com/docker/docker/client" bclient "github.com/moby/buildkit/client" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth/authprovider" @@ -192,63 +194,23 @@ func (s *composeService) getLocalImagesDigests(ctx context.Context, project *typ } func (s *composeService) doBuild(ctx context.Context, project *types.Project, opts map[string]build.Options, mode string) (map[string]string, error) { - info, err := s.apiClient.Info(ctx) - if err != nil { - return nil, err - } - - if info.OSType == "windows" { - // no support yet for Windows container builds in Buildkit - // https://docs.docker.com/develop/develop-images/build_enhancements/#limitations - err := s.windowsBuild(opts, mode) - return nil, WrapCategorisedComposeError(err, BuildFailure) - } if len(opts) == 0 { return nil, nil } - const drivername = "default" - - d, err := driver.GetDriver(ctx, drivername, nil, s.apiClient, s.configFile, nil, nil, "", nil, nil, project.WorkingDir) + dockerCli, err := command.NewDockerCli() if err != nil { return nil, err } - driverInfo := []build.DriverInfo{ - { - Name: "default", - Driver: d, - }, - } - - // Progress needs its own context that lives longer than the - // build one otherwise it won't read all the messages from - // build and will lock - progressCtx, cancel := context.WithCancel(context.Background()) - defer cancel() - w := xprogress.NewPrinter(progressCtx, os.Stdout, mode) - - // We rely on buildx "docker" builder integrated in docker engine, so don't need a DockerAPI here - response, err := build.Build(ctx, driverInfo, opts, nil, nil, w) - errW := w.Wait() - if err == nil { - err = errW - } + err = dockerCli.Initialize(flags.NewClientOptions(), command.WithInitializeClient(func(cli *command.DockerCli) (client.APIClient, error) { + return s.apiClient, nil + })) if err != nil { - return nil, WrapCategorisedComposeError(err, BuildFailure) + return nil, err } - - imagesBuilt := map[string]string{} - for name, img := range response { - if img == nil || len(img.ExporterResponse) == 0 { - continue - } - digest, ok := img.ExporterResponse["containerimage.digest"] - if !ok { - continue - } - imagesBuilt[name] = digest + if buildkitEnabled, err := command.BuildKitEnabled(dockerCli.ServerInfo()); err != nil || !buildkitEnabled { + return s.doBuildClassic(ctx, dockerCli, opts) } - - return imagesBuilt, err + return s.doBuildBuildkit(ctx, project, opts, mode) } func (s *composeService) toBuildOptions(project *types.Project, service types.ServiceConfig, imageTag string) (build.Options, error) { diff --git a/pkg/compose/build_buildkit.go b/pkg/compose/build_buildkit.go new file mode 100644 index 00000000000..39e836a5854 --- /dev/null +++ b/pkg/compose/build_buildkit.go @@ -0,0 +1,72 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "os" + + "github.com/compose-spec/compose-go/types" + "github.com/docker/buildx/build" + "github.com/docker/buildx/driver" + xprogress "github.com/docker/buildx/util/progress" +) + +func (s *composeService) doBuildBuildkit(ctx context.Context, project *types.Project, opts map[string]build.Options, mode string) (map[string]string, error) { + const drivername = "default" + d, err := driver.GetDriver(ctx, drivername, nil, s.apiClient, s.configFile, nil, nil, "", nil, nil, project.WorkingDir) + if err != nil { + return nil, err + } + driverInfo := []build.DriverInfo{ + { + Name: drivername, + Driver: d, + }, + } + + // Progress needs its own context that lives longer than the + // build one otherwise it won't read all the messages from + // build and will lock + progressCtx, cancel := context.WithCancel(context.Background()) + defer cancel() + w := xprogress.NewPrinter(progressCtx, os.Stdout, mode) + + // We rely on buildx "docker" builder integrated in docker engine, so don't need a DockerAPI here + response, err := build.Build(ctx, driverInfo, opts, nil, nil, w) + errW := w.Wait() + if err == nil { + err = errW + } + if err != nil { + return nil, WrapCategorisedComposeError(err, BuildFailure) + } + + imagesBuilt := map[string]string{} + for name, img := range response { + if img == nil || len(img.ExporterResponse) == 0 { + continue + } + digest, ok := img.ExporterResponse["containerimage.digest"] + if !ok { + continue + } + imagesBuilt[name] = digest + } + + return imagesBuilt, err +} diff --git a/pkg/compose/build_classic.go b/pkg/compose/build_classic.go new file mode 100644 index 00000000000..dbe976f5eeb --- /dev/null +++ b/pkg/compose/build_classic.go @@ -0,0 +1,273 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + + buildx "github.com/docker/buildx/build" + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/command/image/build" + dockertypes "github.com/docker/docker/api/types" + "github.com/docker/docker/cli" + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/idtools" + "github.com/docker/docker/pkg/jsonmessage" + "github.com/docker/docker/pkg/progress" + "github.com/docker/docker/pkg/streamformatter" + "github.com/docker/docker/pkg/urlutil" + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" +) + +func (s *composeService) doBuildClassic(ctx context.Context, dockerCli *command.DockerCli, opts map[string]buildx.Options) (map[string]string, error) { + var nameDigests = make(map[string]string) + var errs error + for name, o := range opts { + digest, err := doBuildClassicSimpleImage(ctx, dockerCli, o) + if err != nil { + errs = multierror.Append(errs, err).ErrorOrNil() + } + nameDigests[name] = digest + } + + return nameDigests, errs +} + +// nolint: gocyclo +func doBuildClassicSimpleImage(ctx context.Context, dockerCli *command.DockerCli, options buildx.Options) (string, error) { + var ( + buildCtx io.ReadCloser + dockerfileCtx io.ReadCloser + contextDir string + tempDir string + relDockerfile string + progBuff io.Writer + buildBuff io.Writer + remote string + + err error + ) + + dockerfileName := options.Inputs.DockerfilePath + specifiedContext := options.Inputs.ContextPath + progBuff = dockerCli.Out() + buildBuff = dockerCli.Out() + if options.ImageIDFile != "" { + // Avoid leaving a stale file if we eventually fail + if err := os.Remove(options.ImageIDFile); err != nil && !os.IsNotExist(err) { + return "", errors.Wrap(err, "Removing image ID file") + } + } + + switch { + case isLocalDir(specifiedContext): + contextDir, relDockerfile, err = build.GetContextFromLocalDir(specifiedContext, dockerfileName) + if err == nil && strings.HasPrefix(relDockerfile, ".."+string(filepath.Separator)) { + // Dockerfile is outside of build-context; read the Dockerfile and pass it as dockerfileCtx + dockerfileCtx, err = os.Open(dockerfileName) + if err != nil { + return "", errors.Errorf("unable to open Dockerfile: %v", err) + } + defer dockerfileCtx.Close() // nolint:errcheck + } + case urlutil.IsGitURL(specifiedContext): + tempDir, relDockerfile, err = build.GetContextFromGitURL(specifiedContext, dockerfileName) + case urlutil.IsURL(specifiedContext): + buildCtx, relDockerfile, err = build.GetContextFromURL(progBuff, specifiedContext, dockerfileName) + default: + return "", errors.Errorf("unable to prepare context: path %q not found", specifiedContext) + } + + if err != nil { + return "", errors.Errorf("unable to prepare context: %s", err) + } + + if tempDir != "" { + defer os.RemoveAll(tempDir) // nolint:errcheck + contextDir = tempDir + } + + // read from a directory into tar archive + if buildCtx == nil { + excludes, err := build.ReadDockerignore(contextDir) + if err != nil { + return "", err + } + + if err := build.ValidateContextDirectory(contextDir, excludes); err != nil { + return "", errors.Errorf("error checking context: '%s'.", err) + } + + // And canonicalize dockerfile name to a platform-independent one + relDockerfile = archive.CanonicalTarNameForPath(relDockerfile) + + excludes = build.TrimBuildFilesFromExcludes(excludes, relDockerfile, false) + buildCtx, err = archive.TarWithOptions(contextDir, &archive.TarOptions{ + ExcludePatterns: excludes, + ChownOpts: &idtools.Identity{UID: 0, GID: 0}, + }) + if err != nil { + return "", err + } + } + + // replace Dockerfile if it was added from stdin or a file outside the build-context, and there is archive context + if dockerfileCtx != nil && buildCtx != nil { + buildCtx, relDockerfile, err = build.AddDockerfileToBuildContext(dockerfileCtx, buildCtx) + if err != nil { + return "", err + } + } + + buildCtx, err = build.Compress(buildCtx) + if err != nil { + return "", err + } + + // Setup an upload progress bar + progressOutput := streamformatter.NewProgressOutput(progBuff) + if !dockerCli.Out().IsTerminal() { + progressOutput = &lastProgressOutput{output: progressOutput} + } + + // if up to this point nothing has set the context then we must have another + // way for sending it(streaming) and set the context to the Dockerfile + if dockerfileCtx != nil && buildCtx == nil { + buildCtx = dockerfileCtx + } + + var body io.Reader + if buildCtx != nil { + body = progress.NewProgressReader(buildCtx, progressOutput, 0, "", "Sending build context to Docker daemon") + } + + configFile := dockerCli.ConfigFile() + creds, _ := configFile.GetAllCredentials() + authConfigs := make(map[string]dockertypes.AuthConfig, len(creds)) + for k, auth := range creds { + authConfigs[k] = dockertypes.AuthConfig(auth) + } + buildOptions := imageBuildOptions(options) + buildOptions.Version = dockertypes.BuilderV1 + buildOptions.Dockerfile = relDockerfile + buildOptions.AuthConfigs = authConfigs + buildOptions.RemoteContext = remote + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + response, err := dockerCli.Client().ImageBuild(ctx, body, buildOptions) + if err != nil { + return "", err + } + defer response.Body.Close() // nolint:errcheck + + imageID := "" + aux := func(msg jsonmessage.JSONMessage) { + var result dockertypes.BuildResult + if err := json.Unmarshal(*msg.Aux, &result); err != nil { + fmt.Fprintf(dockerCli.Err(), "Failed to parse aux message: %s", err) + } else { + imageID = result.ID + } + } + + err = jsonmessage.DisplayJSONMessagesStream(response.Body, buildBuff, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), aux) + if err != nil { + if jerr, ok := err.(*jsonmessage.JSONError); ok { + // If no error code is set, default to 1 + if jerr.Code == 0 { + jerr.Code = 1 + } + return "", cli.StatusError{Status: jerr.Message, StatusCode: jerr.Code} + } + return "", err + } + + // Windows: show error message about modified file permissions if the + // daemon isn't running Windows. + if response.OSType != "windows" && runtime.GOOS == "windows" { + // if response.OSType != "windows" && runtime.GOOS == "windows" && !options.quiet { + fmt.Fprintln(dockerCli.Out(), "SECURITY WARNING: You are building a Docker "+ + "image from Windows against a non-Windows Docker host. All files and "+ + "directories added to build context will have '-rwxr-xr-x' permissions. "+ + "It is recommended to double check and reset permissions for sensitive "+ + "files and directories.") + } + + if options.ImageIDFile != "" { + if imageID == "" { + return "", errors.Errorf("Server did not provide an image ID. Cannot write %s", options.ImageIDFile) + } + if err := ioutil.WriteFile(options.ImageIDFile, []byte(imageID), 0666); err != nil { + return "", err + } + } + + return imageID, nil +} + +func isLocalDir(c string) bool { + _, err := os.Stat(c) + return err == nil +} + +func imageBuildOptions(options buildx.Options) dockertypes.ImageBuildOptions { + return dockertypes.ImageBuildOptions{ + Tags: options.Tags, + NoCache: options.NoCache, + PullParent: options.Pull, + BuildArgs: toMapStringStringPtr(options.BuildArgs), + Labels: options.Labels, + NetworkMode: options.NetworkMode, + ExtraHosts: options.ExtraHosts, + Target: options.Target, + } +} + +func toMapStringStringPtr(source map[string]string) map[string]*string { + dest := make(map[string]*string) + for k, v := range source { + v := v + dest[k] = &v + } + return dest +} + +// lastProgressOutput is the same as progress.Output except +// that it only output with the last update. It is used in +// non terminal scenarios to suppress verbose messages +type lastProgressOutput struct { + output progress.Output +} + +// WriteProgress formats progress information from a ProgressReader. +func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error { + if !prog.LastUpdate { + return nil + } + + return out.output.WriteProgress(prog) +} diff --git a/pkg/compose/build_win.go b/pkg/compose/build_win.go deleted file mode 100644 index a38b721c57a..00000000000 --- a/pkg/compose/build_win.go +++ /dev/null @@ -1,28 +0,0 @@ -/* - Copyright 2020 Docker Compose CLI authors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package compose - -import ( - "github.com/docker/buildx/build" - - "github.com/docker/compose/v2/pkg/api" -) - -func (s *composeService) windowsBuild(opts map[string]build.Options, mode string) error { - // FIXME copy/paste or reuse code from https://github.com/docker/cli/blob/master/cli/command/image/build.go - return api.ErrNotImplemented -}