From b8f41a5f56d65ee0b7964424c2818f93473b1c69 Mon Sep 17 00:00:00 2001 From: Yoriyasu Yano <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Aug 2021 12:50:00 -0500 Subject: [PATCH 1/7] Support multiarch docker image builds --- modules/docker/build.go | 99 +++++++++++++++++++++++++++++++----- modules/docker/build_test.go | 18 +++++++ modules/docker/push.go | 25 +++++++++ 3 files changed, 130 insertions(+), 12 deletions(-) create mode 100644 modules/docker/push.go diff --git a/modules/docker/build.go b/modules/docker/build.go index c1025d2b9..c31db19ed 100644 --- a/modules/docker/build.go +++ b/modules/docker/build.go @@ -1,9 +1,12 @@ package docker import ( + "strings" + "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/shell" "github.com/gruntwork-io/terratest/modules/testing" + "github.com/hashicorp/go-multierror" "github.com/stretchr/testify/require" ) @@ -18,6 +21,27 @@ type BuildOptions struct { // Target build arg to pass to the 'docker build' command Target string + // All architectures to target in a multiarch build. Configuring this variable will cause terratest to use docker + // buildx to construct multiarch images. + // You can read more about multiarch docker builds in the official documentation for buildx: + // https://docs.docker.com/buildx/working-with-buildx/ + // NOTE: This list does not automatically include the current platform. For example, if you are building images on + // an Apple Silicon based MacBook, and you configure this variable to []string{"linux/amd64"} to build an amd64 + // image, the buildx command will not automatically include linux/arm64 - you must include that explicitly. + Architectures []string + + // Whether or not to push images directly to the registry on build. Note that for multiarch images (Architectures is + // not empty), this must be true to ensure availability of all architectures - only the image for the current + // platform will be loaded into the daemon (due to a limitation of the docker daemon), so you won't be able to run a + // `docker push` command later to push the multiarch image. + // See https://github.com/moby/moby/pull/38738 for more info on the limitation of multiarch images in docker daemon. + Push bool + + // Whether or not to load the image into the docker daemon at the end of a multiarch build so that it can be used + // locally. Note that this is only used when Architectures is set, and assumes the current architecture is already + // included in the Architectures list. + Load bool + // Custom CLI options that will be passed as-is to the 'docker build' command. This is an "escape hatch" that allows // Terratest to not have to support every single command-line option offered by the 'docker build' command, and // solely focus on the most important ones. @@ -37,25 +61,77 @@ func Build(t testing.TestingT, path string, options *BuildOptions) { func BuildE(t testing.TestingT, path string, options *BuildOptions) error { options.Logger.Logf(t, "Running 'docker build' in %s", path) - args, err := formatDockerBuildArgs(path, options) - if err != nil { - return err - } - cmd := shell.Command{ Command: "docker", - Args: args, + Args: formatDockerBuildArgs(path, options), Logger: options.Logger, } - _, buildErr := shell.RunCommandAndGetOutputE(t, cmd) - return buildErr + if err := shell.RunCommandE(t, cmd); err != nil { + return err + } + + // For non multiarch images, we need to call docker push for each tag since build does not have a push option like + // buildx. + if len(options.Architectures) == 0 && options.Push { + var errorsOccurred = new(multierror.Error) + for _, tag := range options.Tags { + if err := PushE(t, options.Logger, tag); err != nil { + options.Logger.Logf(t, "ERROR: error pushing tag %s", tag) + errorsOccurred = multierror.Append(err) + } + } + return errorsOccurred.ErrorOrNil() + } + + // For multiarch images, if a load is requested call the load command to export the built image into the daemon. + if len(options.Architectures) > 0 && options.Load { + loadCmd := shell.Command{ + Command: "docker", + Args: formatDockerBuildxLoadArgs(path, options), + Logger: options.Logger, + } + return shell.RunCommandE(t, loadCmd) + } + + return nil } // formatDockerBuildArgs formats the arguments for the 'docker build' command. -func formatDockerBuildArgs(path string, options *BuildOptions) ([]string, error) { - args := []string{"build"} +func formatDockerBuildArgs(path string, options *BuildOptions) []string { + args := []string{} + + if len(options.Architectures) > 0 { + args = append( + args, + "buildx", + "build", + "--platform", + strings.Join(options.Architectures, ","), + ) + if options.Push { + args = append(args, "--push") + } + } else { + args = append(args, "build") + } + + return append(args, formatDockerBuildBaseArgs(path, options)...) +} +// formatDockerBuildxLoadArgs formats the arguments for calling load on the 'docker buildx' command. +func formatDockerBuildxLoadArgs(path string, options *BuildOptions) []string { + args := []string{ + "buildx", + "build", + "--load", + } + return append(args, formatDockerBuildBaseArgs(path, options)...) +} + +// formatDockerBuildBaseArgs formats the common args for the build command, both for `build` and `buildx`. +func formatDockerBuildBaseArgs(path string, options *BuildOptions) []string { + args := []string{} for _, tag := range options.Tags { args = append(args, "--tag", tag) } @@ -71,6 +147,5 @@ func formatDockerBuildArgs(path string, options *BuildOptions) ([]string, error) args = append(args, options.OtherOptions...) args = append(args, path) - - return args, nil + return args } diff --git a/modules/docker/build_test.go b/modules/docker/build_test.go index 396ef7bd6..94319c7a0 100644 --- a/modules/docker/build_test.go +++ b/modules/docker/build_test.go @@ -24,6 +24,24 @@ func TestBuild(t *testing.T) { require.Contains(t, out, text) } +func TestBuildMultiArch(t *testing.T) { + t.Parallel() + + tag := "gruntwork-io/test-image:v1" + text := "Hello, World!" + + options := &BuildOptions{ + Tags: []string{tag}, + BuildArgs: []string{fmt.Sprintf("text=%s", text)}, + Architectures: []string{"linux/arm64", "linux/amd64"}, + Load: true, + } + + Build(t, "../../test/fixtures/docker", options) + out := Run(t, tag, &RunOptions{Remove: true}) + require.Contains(t, out, text) +} + func TestBuildWithTarget(t *testing.T) { t.Parallel() diff --git a/modules/docker/push.go b/modules/docker/push.go new file mode 100644 index 000000000..af0659b54 --- /dev/null +++ b/modules/docker/push.go @@ -0,0 +1,25 @@ +package docker + +import ( + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/shell" + "github.com/gruntwork-io/terratest/modules/testing" + "github.com/stretchr/testify/require" +) + +// Push runs the 'docker push' command to push the given tag. This will fail the test if there are any errors. +func Push(t testing.TestingT, logger *logger.Logger, tag string) { + require.NoError(t, PushE(t, logger, tag)) +} + +// PushE runs the 'docker push' command to push the given tag. +func PushE(t testing.TestingT, logger *logger.Logger, tag string) error { + logger.Logf(t, "Running 'docker push' for tag %s", tag) + + cmd := shell.Command{ + Command: "docker", + Args: []string{"push", tag}, + Logger: logger, + } + return shell.RunCommandE(t, cmd) +} From f6847d498ca5ac005a2657657a98c7a8ad17a885 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Aug 2021 13:49:17 -0500 Subject: [PATCH 2/7] Make sure to install docker buildx plugin in circleci --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1cf3eccc2..fb2b0c3eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,6 +53,14 @@ install_gruntwork_utils: &install_gruntwork_utils sudo ln -s /usr/local/go/bin/go /usr/bin/go echo "The installed version of Go is now $(go version)" +install_docker_buildx: &install_docker_buildx + name: install docker buildx + command: | + curl -sLO https://github.com/docker/buildx/releases/download/v0.6.1/buildx-v0.6.1.linux-amd64 + mkdir -p ~/.docker/cli-plugins + mv buildx-v0.6.1.linux-amd64 ~/.docker/cli-plugins/docker-buildx + chmod a+x ~/.docker/cli-plugins/docker-buildx + configure_environment_for_gcp: &configure_environment_for_gcp name: configure environment for gcp command: | @@ -143,6 +151,8 @@ jobs: - run: <<: *install_gruntwork_utils + - run: + <<: *install_docker_buildx # The weird way you have to set PATH in Circle 2.0 - run: echo 'export PATH=$HOME/.local/bin:$HOME/terraform:$HOME/packer:$PATH' >> $BASH_ENV From 0f024b5b360b18782534f84f77c431a64b4d4028 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Aug 2021 14:50:18 -0500 Subject: [PATCH 3/7] Use newer ubuntu and docker for buildx --- .circleci/config.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index fb2b0c3eb..97932381c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,6 +15,13 @@ env: &env BIN_BUILD_PARALLELISM: 3 defaults: &defaults + machine: + enabled: true + image: "ubuntu-2004:202107-02" + <<: *env + +# minikube setup requires ubuntu16.04 +minikube_defaults: &minikube_defaults machine: enabled: true image: "ubuntu-1604:201903-01" @@ -61,6 +68,10 @@ install_docker_buildx: &install_docker_buildx mv buildx-v0.6.1.linux-amd64 ~/.docker/cli-plugins/docker-buildx chmod a+x ~/.docker/cli-plugins/docker-buildx + # Verify buildx is available + docker buildx --help + exit 1 + configure_environment_for_gcp: &configure_environment_for_gcp name: configure environment for gcp command: | @@ -219,7 +230,7 @@ jobs: path: /tmp/logs kubernetes_test: - <<: *defaults + <<: *minikube_defaults steps: - attach_workspace: at: /home/circleci @@ -255,7 +266,7 @@ jobs: helm_test: - <<: *defaults + <<: *minikube_defaults steps: - attach_workspace: at: /home/circleci From 05e28d1af9c72b51a8b34316ff8f11954b19cafc Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Aug 2021 14:58:19 -0500 Subject: [PATCH 4/7] Remove test --- .circleci/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 97932381c..ffc439a65 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -70,7 +70,6 @@ install_docker_buildx: &install_docker_buildx # Verify buildx is available docker buildx --help - exit 1 configure_environment_for_gcp: &configure_environment_for_gcp name: configure environment for gcp From f5020d832d7fb2a00650dfe1d1b91894a3f322ae Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Aug 2021 16:13:22 -0500 Subject: [PATCH 5/7] Fix build --- .circleci/config.yml | 2 +- modules/aws/ecr_test.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ffc439a65..efdb636c9 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -69,7 +69,7 @@ install_docker_buildx: &install_docker_buildx chmod a+x ~/.docker/cli-plugins/docker-buildx # Verify buildx is available - docker buildx --help + docker buildx create --use configure_environment_for_gcp: &configure_environment_for_gcp name: configure environment for gcp diff --git a/modules/aws/ecr_test.go b/modules/aws/ecr_test.go index 4ed559d66..8920d6c71 100644 --- a/modules/aws/ecr_test.go +++ b/modules/aws/ecr_test.go @@ -1,9 +1,12 @@ package aws import ( + "fmt" + "strings" "testing" "github.com/aws/aws-sdk-go/aws" + "github.com/gruntwork-io/terratest/modules/random" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -12,7 +15,7 @@ func TestEcrRepo(t *testing.T) { t.Parallel() region := GetRandomStableRegion(t, nil, nil) - repo1, err := CreateECRRepoE(t, region, "terratest") + repo1, err := CreateECRRepoE(t, region, fmt.Sprintf("terratest%s", strings.ToLower(random.UniqueId()))) defer DeleteECRRepo(t, region, repo1) require.NoError(t, err) From 740418b1f76128f0c69d50164bb7c0f5605f7a28 Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Aug 2021 17:40:50 -0500 Subject: [PATCH 6/7] Make sure to check against the right name --- modules/aws/ecr_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/aws/ecr_test.go b/modules/aws/ecr_test.go index 8920d6c71..e006cc42c 100644 --- a/modules/aws/ecr_test.go +++ b/modules/aws/ecr_test.go @@ -15,14 +15,14 @@ func TestEcrRepo(t *testing.T) { t.Parallel() region := GetRandomStableRegion(t, nil, nil) - repo1, err := CreateECRRepoE(t, region, fmt.Sprintf("terratest%s", strings.ToLower(random.UniqueId()))) + ecrRepoName := fmt.Sprintf("terratest%s", strings.ToLower(random.UniqueId())) + repo1, err := CreateECRRepoE(t, region, ecrRepoName) defer DeleteECRRepo(t, region, repo1) - require.NoError(t, err) - assert.Equal(t, "terratest", aws.StringValue(repo1.RepositoryName)) - repo2, err := GetECRRepoE(t, region, "terratest") + assert.Equal(t, "terratest", aws.StringValue(repo1.RepositoryName)) + repo2, err := GetECRRepoE(t, region, ecrRepoName) require.NoError(t, err) assert.Equal(t, "terratest", aws.StringValue(repo2.RepositoryName)) } From fd6e979e037c8ccc5a2e933f1d313d8b9994fd8b Mon Sep 17 00:00:00 2001 From: yorinasub17 <430092+yorinasub17@users.noreply.github.com> Date: Thu, 5 Aug 2021 18:57:16 -0500 Subject: [PATCH 7/7] Actually fix ecr repo tests --- modules/aws/ecr_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/aws/ecr_test.go b/modules/aws/ecr_test.go index e006cc42c..b21652192 100644 --- a/modules/aws/ecr_test.go +++ b/modules/aws/ecr_test.go @@ -20,9 +20,9 @@ func TestEcrRepo(t *testing.T) { defer DeleteECRRepo(t, region, repo1) require.NoError(t, err) - assert.Equal(t, "terratest", aws.StringValue(repo1.RepositoryName)) + assert.Equal(t, ecrRepoName, aws.StringValue(repo1.RepositoryName)) repo2, err := GetECRRepoE(t, region, ecrRepoName) require.NoError(t, err) - assert.Equal(t, "terratest", aws.StringValue(repo2.RepositoryName)) + assert.Equal(t, ecrRepoName, aws.StringValue(repo2.RepositoryName)) }