From 1d7be81d35a602c94256bc672f75f2660ed3ff44 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 15 Mar 2023 17:19:55 +0100 Subject: [PATCH] use `build` as common API for build scenarios Signed-off-by: Nicolas De Loof --- pkg/compose/build.go | 120 +++++++++++++++--------------- pkg/compose/build_test.go | 121 +++++++++++++++++++++++++++++++ pkg/compose/dependencies_test.go | 36 ++++----- pkg/compose/watch.go | 5 ++ pkg/e2e/build_test.go | 6 +- 5 files changed, 209 insertions(+), 79 deletions(-) create mode 100644 pkg/compose/build_test.go diff --git a/pkg/compose/build.go b/pkg/compose/build.go index cbeb5893e27..537771c0bdb 100644 --- a/pkg/compose/build.go +++ b/pkg/compose/build.go @@ -69,41 +69,11 @@ func (s *composeService) build(ctx context.Context, project *types.Project, opti return nil } imageName := api.GetImageNameOrDefault(service, project.Name) - buildOptions, err := s.toBuildOptions(project, service, imageName, options.SSHs) + buildOptions, err := s.toBuildOptions(project, service, imageName, options) if err != nil { return err } - buildOptions.Pull = options.Pull buildOptions.BuildArgs = mergeArgs(buildOptions.BuildArgs, args) - buildOptions.NoCache = options.NoCache - buildOptions.CacheFrom, err = buildflags.ParseCacheEntry(service.Build.CacheFrom) - if err != nil { - return err - } - if len(service.Build.AdditionalContexts) > 0 { - buildOptions.Inputs.NamedContexts = toBuildContexts(service.Build.AdditionalContexts) - } - for _, image := range service.Build.CacheFrom { - buildOptions.CacheFrom = append(buildOptions.CacheFrom, bclient.CacheOptionsEntry{ - Type: "registry", - Attrs: map[string]string{"ref": image}, - }) - } - buildOptions.Exports = []bclient.ExportEntry{{ - Type: "docker", - Attrs: map[string]string{ - "load": "true", - "push": fmt.Sprint(options.Push), - }, - }} - if len(buildOptions.Platforms) > 1 { - buildOptions.Exports = []bclient.ExportEntry{{ - Type: "image", - Attrs: map[string]string{ - "push": fmt.Sprint(options.Push), - }, - }} - } opts := map[string]build.Options{imageName: buildOptions} ids, err := s.doBuild(ctx, project, opts, options.Progress) if err != nil { @@ -146,11 +116,14 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. if quietPull { mode = xprogress.PrinterModeQuiet } - opts, err := s.getBuildOptions(project, images) + + err = s.prepareProjectForBuild(project, images) if err != nil { return err } - builtImages, err := s.doBuild(ctx, project, opts, mode) + builtImages, err := s.build(ctx, project, api.BuildOptions{ + Progress: mode, + }) if err != nil { return err } @@ -172,37 +145,45 @@ func (s *composeService) ensureImagesExists(ctx context.Context, project *types. return nil } -func (s *composeService) getBuildOptions(project *types.Project, images map[string]string) (map[string]build.Options, error) { - opts := map[string]build.Options{} - for _, service := range project.Services { +func (s *composeService) prepareProjectForBuild(project *types.Project, images map[string]string) error { + platform := project.Environment["DOCKER_DEFAULT_PLATFORM"] + for i, service := range project.Services { if service.Image == "" && service.Build == nil { - return nil, fmt.Errorf("invalid service %q. Must specify either image or build", service.Name) + return fmt.Errorf("invalid service %q. Must specify either image or build", service.Name) } + if service.Build == nil { + continue + } + imageName := api.GetImageNameOrDefault(service, project.Name) + service.Image = imageName + _, localImagePresent := images[imageName] + if localImagePresent && service.PullPolicy != types.PullPolicyBuild { + service.Build = nil + project.Services[i] = service + continue + } - if service.Build != nil { - if localImagePresent && service.PullPolicy != types.PullPolicyBuild { - continue + if platform != "" { + if len(service.Build.Platforms) > 0 && !utils.StringContains(service.Build.Platforms, platform) { + return fmt.Errorf("service %q build.platforms does not support value set by DOCKER_DEFAULT_PLATFORM: %s", service.Name, platform) } - opt, err := s.toBuildOptions(project, service, imageName, []types.SSHKey{}) - if err != nil { - return nil, err - } - opt.Exports = []bclient.ExportEntry{{ - Type: "docker", - Attrs: map[string]string{ - "load": "true", - }, - }} - if opt.Platforms, err = useDockerDefaultOrServicePlatform(project, service, true); err != nil { - opt.Platforms = []specs.Platform{} + service.Platform = platform + } + + if service.Platform == "" { + // let builder to build for default platform + service.Build.Platforms = nil + } else { + if len(service.Build.Platforms) > 0 && !utils.StringContains(service.Build.Platforms, service.Platform) { + return fmt.Errorf("service %q build configuration does not support platform: %s", service.Name, platform) } - opts[imageName] = opt - continue + service.Build.Platforms = []string{service.Platform} } + project.Services[i] = service } - return opts, nil + return nil } func (s *composeService) getLocalImagesDigests(ctx context.Context, project *types.Project) (map[string]string, error) { @@ -243,7 +224,7 @@ func (s *composeService) doBuild(ctx context.Context, project *types.Project, op return s.doBuildBuildkit(ctx, opts, mode) } -func (s *composeService) toBuildOptions(project *types.Project, service types.ServiceConfig, imageTag string, sshKeys []types.SSHKey) (build.Options, error) { +func (s *composeService) toBuildOptions(project *types.Project, service types.ServiceConfig, imageTag string, options api.BuildOptions) (build.Options, error) { var tags []string tags = append(tags, imageTag) @@ -272,8 +253,8 @@ func (s *composeService) toBuildOptions(project *types.Project, service types.Se sessionConfig := []session.Attachable{ authprovider.NewDockerAuthProvider(s.configFile()), } - if len(sshKeys) > 0 || len(service.Build.SSH) > 0 { - sshAgentProvider, err := sshAgentProvider(append(service.Build.SSH, sshKeys...)) + if len(options.SSHs) > 0 || len(service.Build.SSH) > 0 { + sshAgentProvider, err := sshAgentProvider(append(service.Build.SSH, options.SSHs...)) if err != nil { return build.Options{}, err } @@ -298,20 +279,37 @@ func (s *composeService) toBuildOptions(project *types.Project, service types.Se imageLabels := getImageBuildLabels(project, service) + exports := []bclient.ExportEntry{{ + Type: "docker", + Attrs: map[string]string{ + "load": "true", + "push": fmt.Sprint(options.Push), + }, + }} + if len(service.Build.Platforms) > 1 { + exports = []bclient.ExportEntry{{ + Type: "image", + Attrs: map[string]string{ + "push": fmt.Sprint(options.Push), + }, + }} + } + return build.Options{ Inputs: build.Inputs{ ContextPath: service.Build.Context, DockerfileInline: service.Build.DockerfileInline, DockerfilePath: dockerFilePath(service.Build.Context, service.Build.Dockerfile), + NamedContexts: toBuildContexts(service.Build.AdditionalContexts), }, CacheFrom: cacheFrom, CacheTo: cacheTo, - NoCache: service.Build.NoCache, - Pull: service.Build.Pull, + NoCache: service.Build.NoCache || options.NoCache, + Pull: service.Build.Pull || options.Pull, BuildArgs: buildArgs, Tags: tags, Target: service.Build.Target, - Exports: []bclient.ExportEntry{{Type: "image", Attrs: map[string]string{}}}, + Exports: exports, Platforms: plats, Labels: imageLabels, NetworkMode: service.Build.Network, diff --git a/pkg/compose/build_test.go b/pkg/compose/build_test.go new file mode 100644 index 00000000000..145118a5c1b --- /dev/null +++ b/pkg/compose/build_test.go @@ -0,0 +1,121 @@ +/* + Copyright 2020 Docker Compose CLI authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package compose + +import ( + "testing" + + "github.com/compose-spec/compose-go/types" + "gotest.tools/v3/assert" +) + +func TestPrepareProjectForBuild(t *testing.T) { + t.Run("build service platform", func(t *testing.T) { + project := types.Project{ + Services: []types.ServiceConfig{ + { + Name: "test", + Image: "foo", + Build: &types.BuildConfig{ + Context: ".", + Platforms: []string{ + "linux/amd64", + "linux/arm64", + "alice/32", + }, + }, + Platform: "alice/32", + }, + }, + } + + s := &composeService{} + err := s.prepareProjectForBuild(&project, nil) + assert.NilError(t, err) + assert.DeepEqual(t, project.Services[0].Build.Platforms, types.StringList{"alice/32"}) + }) + + t.Run("build DOCKER_DEFAULT_PLATFORM", func(t *testing.T) { + project := types.Project{ + Environment: map[string]string{ + "DOCKER_DEFAULT_PLATFORM": "linux/amd64", + }, + Services: []types.ServiceConfig{ + { + Name: "test", + Image: "foo", + Build: &types.BuildConfig{ + Context: ".", + Platforms: []string{ + "linux/amd64", + "linux/arm64", + }, + }, + }, + }, + } + + s := &composeService{} + err := s.prepareProjectForBuild(&project, nil) + assert.NilError(t, err) + assert.DeepEqual(t, project.Services[0].Build.Platforms, types.StringList{"linux/amd64"}) + }) + + t.Run("skip existing image", func(t *testing.T) { + project := types.Project{ + Services: []types.ServiceConfig{ + { + Name: "test", + Image: "foo", + Build: &types.BuildConfig{ + Context: ".", + }, + }, + }, + } + + s := &composeService{} + err := s.prepareProjectForBuild(&project, map[string]string{"foo": "exists"}) + assert.NilError(t, err) + assert.Check(t, project.Services[0].Build == nil) + }) + + t.Run("unsupported build platform", func(t *testing.T) { + project := types.Project{ + Environment: map[string]string{ + "DOCKER_DEFAULT_PLATFORM": "commodore/64", + }, + Services: []types.ServiceConfig{ + { + Name: "test", + Image: "foo", + Build: &types.BuildConfig{ + Context: ".", + Platforms: []string{ + "linux/amd64", + "linux/arm64", + }, + }, + }, + }, + } + + s := &composeService{} + err := s.prepareProjectForBuild(&project, nil) + assert.Check(t, err != nil) + }) +} diff --git a/pkg/compose/dependencies_test.go b/pkg/compose/dependencies_test.go index 9a958820ad3..1b22cd6033f 100644 --- a/pkg/compose/dependencies_test.go +++ b/pkg/compose/dependencies_test.go @@ -27,24 +27,26 @@ import ( "gotest.tools/v3/assert" ) -var project = types.Project{ - Services: []types.ServiceConfig{ - { - Name: "test1", - DependsOn: map[string]types.ServiceDependency{ - "test2": {}, +func createTestProject() *types.Project { + return &types.Project{ + Services: []types.ServiceConfig{ + { + Name: "test1", + DependsOn: map[string]types.ServiceDependency{ + "test2": {}, + }, }, - }, - { - Name: "test2", - DependsOn: map[string]types.ServiceDependency{ - "test3": {}, + { + Name: "test2", + DependsOn: map[string]types.ServiceDependency{ + "test3": {}, + }, + }, + { + Name: "test3", }, }, - { - Name: "test3", - }, - }, + } } func TestTraversalWithMultipleParents(t *testing.T) { @@ -97,7 +99,7 @@ func TestInDependencyUpCommandOrder(t *testing.T) { t.Cleanup(cancel) var order []string - err := InDependencyOrder(ctx, &project, func(ctx context.Context, service string) error { + err := InDependencyOrder(ctx, createTestProject(), func(ctx context.Context, service string) error { order = append(order, service) return nil }) @@ -110,7 +112,7 @@ func TestInDependencyReverseDownCommandOrder(t *testing.T) { t.Cleanup(cancel) var order []string - err := InReverseDependencyOrder(ctx, &project, func(ctx context.Context, service string) error { + err := InReverseDependencyOrder(ctx, createTestProject(), func(ctx context.Context, service string) error { order = append(order, service) return nil }) diff --git a/pkg/compose/watch.go b/pkg/compose/watch.go index 6b0588c1afa..7b0057f2ec0 100644 --- a/pkg/compose/watch.go +++ b/pkg/compose/watch.go @@ -54,6 +54,11 @@ func (s *composeService) Watch(ctx context.Context, project *types.Project, serv needRebuild := make(chan string) needSync := make(chan api.CopyOptions, 5) + err := s.prepareProjectForBuild(project, nil) + if err != nil { + return err + } + eg, ctx := errgroup.WithContext(ctx) eg.Go(func() error { clock := clockwork.NewRealClock() diff --git a/pkg/e2e/build_test.go b/pkg/e2e/build_test.go index 101761659cf..98612ce17a1 100644 --- a/pkg/e2e/build_test.go +++ b/pkg/e2e/build_test.go @@ -265,7 +265,11 @@ func TestBuildImageDependencies(t *testing.T) { }) t.Run("BuildKit", func(t *testing.T) { - t.Skip("See https://github.com/docker/compose/issues/9232") + cli := NewParallelCLI(t, WithEnv( + "DOCKER_BUILDKIT=1", + "COMPOSE_FILE=./fixtures/build-dependencies/compose.yaml", + )) + doTest(t, cli) }) }