Merge pull request #9962 from edsantiago/podman_remote_arch

Handle podman-remote --arch, --platform, --os

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b
 	github.com/containernetworking/cni v0.8.1
 	github.com/containernetworking/plugins v0.9.1
-	github.com/containers/buildah v1.20.0
+	github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8
 	github.com/containers/common v0.35.4
 	github.com/containers/conmon v2.0.20+incompatible
 	github.com/containers/image/v5 v5.10.5

go.sum

@@ -174,9 +174,8 @@ github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHV
 github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0=
 github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8=
 github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8=
-github.com/containers/buildah v1.20.0 h1:H8db/d2uSGm947mqjX0Iup6F0T9AnK3kS/ff9RCemZA=
-github.com/containers/buildah v1.20.0/go.mod h1:8V3UBoTKBWU9AxNHb1MAKnZZ9oSoz/IsYyjeymrpl1s=
-github.com/containers/common v0.35.3/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE=
+github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8 h1:RlqbDlfE3+qrq4bNTZG7NVPqCDzfZrgE/yicu0VAykQ=
+github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8/go.mod h1:iowyscoAC5jwNDhs3c5CLGdBZ9FJk5UOoN2I5TdmXFs=
 github.com/containers/common v0.35.4 h1:szyWRncsHkBwCVpu1dkEOXUjkwCetlfcLmKJTwo1Sp8=
 github.com/containers/common v0.35.4/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE=
 github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg=

pkg/api/handlers/compat/images_build.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/containers/buildah"
 	buildahDefine "github.com/containers/buildah/define"
+	"github.com/containers/buildah/pkg/parse"
 	"github.com/containers/buildah/util"
 	"github.com/containers/image/v5/types"
 	"github.com/containers/podman/v3/libpod"
@@ -445,6 +446,17 @@ func BuildImage(w http.ResponseWriter, r *http.Request) {
 		},
 	}
 
+	if len(query.Platform) > 0 {
+		variant := ""
+		buildOptions.OS, buildOptions.Architecture, variant, err = parse.Platform(query.Platform)
+		if err != nil {
+			utils.BadRequest(w, "platform", query.Platform, err)
+			return
+		}
+		buildOptions.SystemContext.OSChoice = buildOptions.OS
+		buildOptions.SystemContext.ArchitectureChoice = buildOptions.Architecture
+		buildOptions.SystemContext.VariantChoice = variant
+	}
 	if _, found := r.URL.Query()["timestamp"]; found {
 		ts := time.Unix(query.Timestamp, 0)
 		buildOptions.Timestamp = &ts

pkg/bindings/images/build.go

@@ -12,6 +12,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"runtime"
 	"strconv"
 	"strings"
 
@@ -190,6 +191,10 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO
 			platform = "linux"
 		}
 		platform += "/" + options.Architecture
+	} else {
+		if len(platform) > 0 {
+			platform += "/" + runtime.GOARCH
+		}
 	}
 	if len(platform) > 0 {
 		params.Set("platform", platform)

test/buildah-bud/buildah-tests.diff

@@ -1,16 +1,16 @@
-From a49a2e48421c6f3bb1a56ae372de1f3d1a45d1f1 Mon Sep 17 00:00:00 2001
+From 47cc8f54d272039bdab389b29c8b2b5bb085e923 Mon Sep 17 00:00:00 2001
 From: Ed Santiago <[email protected]>
 Date: Tue, 9 Feb 2021 17:28:05 -0700
 Subject: [PATCH] tweaks for running buildah tests under podman
 
 Signed-off-by: Ed Santiago <[email protected]>
 ---
- tests/bud.bats     | 26 ++++++++++++++++----------
+ tests/bud.bats     | 22 +++++++++++++---------
  tests/helpers.bash | 28 ++++++++++++++++++++++++----
- 2 files changed, 40 insertions(+), 14 deletions(-)
+ 2 files changed, 37 insertions(+), 13 deletions(-)
 
 diff --git a/tests/bud.bats b/tests/bud.bats
-index cf55d9a4..60cb6f96 100644
+index fe3af27a..d1ccced5 100644
 --- a/tests/bud.bats
 +++ b/tests/bud.bats
 @@ -4,7 +4,7 @@ load helpers
@@ -30,7 +30,7 @@ index cf55d9a4..60cb6f96 100644
    run_buildah 125 bud /tmp/tmpdockerfile/ -t blabla
    check_options_flag_err "-t"
 
-@@ -1416,13 +1417,13 @@ function _test_http() {
+@@ -1436,13 +1437,13 @@ function _test_http() {
  @test "bud with dir for file but no Dockerfile in dir" {
    target=alpine-image
    run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json -t ${target} -f ${TESTSDIR}/bud/empty-dir ${TESTSDIR}/bud/empty-dir
@@ -46,26 +46,15 @@ index cf55d9a4..60cb6f96 100644
  }
 
  @test "bud with ARG before FROM default value" {
-@@ -1834,7 +1835,9 @@ _EOF
-   run_buildah bud --signature-policy ${TESTSDIR}/policy.json --layers -t test-img-2 --build-arg TEST=foo -f Dockerfile4 ${TESTSDIR}/bud/build-arg
-   run_buildah inspect -f '{{.FromImageID}}' test-img-2
-   argsid="$output"
--  [[ "$argsid" != "$initialid" ]]
-+  if [[ "$argsid" == "$initialid" ]]; then
-+      die ".FromImageID of test-img-2 ($argsid) == same as test-img, it should be different"
-+  fi
-
-   # Set the build-arg via an ENV in the local environment and verify that the cached layers are not used
-   export TEST=bar
-@@ -1887,6 +1890,7 @@ _EOF
+@@ -1913,6 +1914,7 @@ _EOF
  }
 
  @test "bud without any arguments should succeed" {
 +  skip "does not work under podman"
    cd ${TESTSDIR}/bud/from-scratch
    run_buildah bud --signature-policy ${TESTSDIR}/policy.json
  }
-@@ -1894,7 +1898,7 @@ _EOF
+@@ -1920,7 +1922,7 @@ _EOF
  @test "bud without any arguments should fail when no Dockerfile exist" {
    cd $(mktemp -d)
    run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json
@@ -74,7 +63,7 @@ index cf55d9a4..60cb6f96 100644
  }
 
  @test "bud with specified context should fail if directory contains no Dockerfile" {
-@@ -1907,16 +1911,17 @@ _EOF
+@@ -1933,16 +1935,17 @@ _EOF
    DIR=$(mktemp -d)
    mkdir -p "$DIR"/Dockerfile
    run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"
@@ -94,7 +83,7 @@ index cf55d9a4..60cb6f96 100644
    DIR=$(mktemp -d)
    echo "FROM alpine" > "$DIR"/Dockerfile
    run_buildah 0 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"/Dockerfile
-@@ -1968,7 +1973,7 @@ _EOF
+@@ -1994,7 +1997,7 @@ _EOF
 
  @test "bud-squash-hardlinks" {
    _prefetch busybox
@@ -103,23 +92,23 @@ index cf55d9a4..60cb6f96 100644
  }
 
  @test "bud with additional directory of devices" {
-@@ -2134,6 +2139,7 @@ _EOF
+@@ -2159,6 +2162,7 @@ _EOF
  }
 
  @test "bud with Containerfile should fail with nonexistent authfile" {
 +  skip "FIXME: podman issue #9572"
    target=alpine-image
    run_buildah 125 bud --authfile /tmp/nonexistent --signature-policy ${TESTSDIR}/policy.json -t ${target} ${TESTSDIR}/bud/containerfile
  }
-@@ -2261,6 +2267,7 @@ EOM
+@@ -2286,6 +2290,7 @@ EOM
  }
 
  @test "bud with encrypted FROM image" {
 +  skip "Too much effort to spin up a local registry"
    _prefetch busybox
    mkdir ${TESTDIR}/tmp
    openssl genrsa -out ${TESTDIR}/tmp/mykey.pem 1024
-@@ -2333,8 +2340,6 @@ EOM
+@@ -2358,8 +2363,6 @@ EOM
    _prefetch alpine
    run_buildah bud --timestamp=0 --quiet --pull=false --signature-policy ${TESTSDIR}/policy.json -t timestamp -f Dockerfile.1 ${TESTSDIR}/bud/cache-stages
    cid=$output
@@ -128,7 +117,7 @@ index cf55d9a4..60cb6f96 100644
    run_buildah inspect --format '{{ .OCIv1.Created }}' timestamp
    expect_output --substring "1970-01-01"
    run_buildah inspect --format '{{ .History }}' timestamp
-@@ -2594,6 +2599,7 @@ _EOF
+@@ -2619,6 +2622,7 @@ _EOF
  }
 
  @test "bud with --arch flag" {
@@ -137,18 +126,18 @@ index cf55d9a4..60cb6f96 100644
    mytmpdir=${TESTDIR}/my-dir
    mkdir -p ${mytmpdir}
 diff --git a/tests/helpers.bash b/tests/helpers.bash
-index b28fd2c3..d42a6b82 100644
+index 99c290af..c5572840 100644
 --- a/tests/helpers.bash
 +++ b/tests/helpers.bash
 @@ -70,7 +70,7 @@ function _prefetch() {
-             mkdir -p ${_BUILDAH_IMAGE_CACHEDIR}
-         fi
+         mkdir -p ${_BUILDAH_IMAGE_CACHEDIR}
+     fi
 
--        local _podman_opts="--root ${TESTDIR}/root --storage-driver ${STORAGE_DRIVER}"
-+        local _podman_opts="--root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER}"
+-    local _podman_opts="--root ${TESTDIR}/root --storage-driver ${STORAGE_DRIVER}"
++    local _podman_opts="--root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER}"
 
-         for img in "$@"; do
-             echo "# [checking for: $img]" >&2
+     for img in "$@"; do
+         echo "# [checking for: $img]" >&2
 @@ -138,15 +138,35 @@ function run_buildah() {
          --retry)         retry=3;        shift;;  # retry network flakes
      esac

test/buildah-bud/run-buildah-bud-tests

@@ -72,7 +72,7 @@ function die() {
 
 # From here on out, any unexpected abort will try to offer helpful hints
 failhint=
-trap 'if [[ $? != 0 ]]; then if [[ -n $failhint ]]; then echo;echo "***************************************";echo $failhint;echo;echo "Please see $BUD_TEST_DIR_REL/README.md for advice";fi;fi' 0
+trap 'if [[ $? != 0 ]]; then if [[ -n $failhint ]]; then echo;echo "***************************************";echo "$failhint";echo;echo "Please see $BUD_TEST_DIR_REL/README.md for advice";fi;fi' 0
 
 # Find the version of buildah we've vendored in, so we can run the right tests
 buildah_version=$(awk "\$1 == \"$BUILDAH_REPO\" { print \$2 }" <go.mod)
@@ -110,10 +110,27 @@ if [[ -n $do_checkout ]]; then
         die "Directory already exists: $buildah_dir"
     fi
 
+    # buildah_version should usually be vX.Y, but sometimes a PR under test
+    # will need a special unreleased version (go calls then "pseudoversions").
+    # In the usual case, we can do a shallow git clone:
+    shallow_checkout="--branch $buildah_version"
+    if [[ $buildah_version =~ .*-.*\.[0-9]{14}-.* ]]; then
+        # ...but with a pseudoversion, we must git-clone the entire repo,
+        # then do a git checkout within it
+        shallow_checkout=
+    fi
+
     failhint="'git clone' failed - this should never happen!"
-    (set -x;git clone -q --branch $buildah_version https://$BUILDAH_REPO $buildah_dir)
+    (set -x;git clone -q $shallow_checkout https://$BUILDAH_REPO $buildah_dir)
 
     cd $buildah_dir
+    if [[ -z $shallow_checkout ]]; then
+        # extract the SHA (rightmost field) from, e.g., v1.2-YYYMMDD-<sha>
+        sha=${buildah_version##*-}
+
+        failhint="'git checkout $sha' failed - this should never happen!"
+        (set -x;git checkout -q $sha)
+    fi
 
     # Give it a recognizable tag; this will be useful if we need to update
     # the set of patches
@@ -123,18 +140,22 @@ if [[ -n $do_checkout ]]; then
     failhint="error building buildah. This should never happen."
     (set -x;make bin/buildah)
 
-    # Apply custom patches. We do this _after_ building, although it shouldn't
-    # matter because these patches should only apply to test scripts.
-    failhint="
-Error applying patch file. This can happen when you vendor in a new buildah."
-    (set -x;git am <$PATCHES)
-
+    # The upcoming patch may fail. Before we try it, create a helper script
+    # for a developer to push a new set of diffs to podman-land.
     failhint=
     sed -e "s,\[BASETAG\],${BASE_TAG},g" \
         -e "s,\[BUILDAHREPO\],${BUILDAH_REPO},g" \
         < ${BUD_TEST_DIR}/make-new-buildah-diffs \
         > make-new-buildah-diffs
     chmod 755 make-new-buildah-diffs
+
+    # Apply custom patches. We do this _after_ building, although it shouldn't
+    # matter because these patches should only apply to test scripts.
+    failhint="
+Error applying patch file. This can happen when you vendor in a new buildah.
+
+Look for '*.rej' files to resolve the conflict(s) manually."
+    (set -x;git am --reject <$PATCHES)
 else
     # Called with --no-checkout
     test -d $buildah_dir || die "Called with --no-checkout, but $buildah_dir does not exist"

test/e2e/build_test.go

@@ -566,4 +566,42 @@ RUN echo hello`, ALPINE)
 		Expect(session.OutputToString()).To(ContainSubstring("(user)"))
 		Expect(session.OutputToString()).To(ContainSubstring("(elapsed)"))
 	})
+
+	It("podman build --arch --os flag", func() {
+		containerfile := `FROM scratch`
+		containerfilePath := filepath.Join(podmanTest.TempDir, "Containerfile")
+		err := ioutil.WriteFile(containerfilePath, []byte(containerfile), 0755)
+		Expect(err).To(BeNil())
+		session := podmanTest.Podman([]string{"build", "--pull-never", "-t", "test", "--arch", "foo", "--os", "bar", "--file", containerfilePath, podmanTest.TempDir})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		inspect := podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Architecture }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal("foo"))
+
+		inspect = podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Os }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal("bar"))
+
+	})
+
+	It("podman build --os windows flag", func() {
+		containerfile := `FROM scratch`
+		containerfilePath := filepath.Join(podmanTest.TempDir, "Containerfile")
+		err := ioutil.WriteFile(containerfilePath, []byte(containerfile), 0755)
+		Expect(err).To(BeNil())
+		session := podmanTest.Podman([]string{"build", "--pull-never", "-t", "test", "--os", "windows", "--file", containerfilePath, podmanTest.TempDir})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+
+		inspect := podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Architecture }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal(runtime.GOARCH))
+
+		inspect = podmanTest.Podman([]string{"image", "inspect", "--format", "{{ .Os }}", "test"})
+		inspect.WaitWithDefaultTimeout()
+		Expect(inspect.OutputToString()).To(Equal("windows"))
+
+	})
 })