Script for multi-arch parallel image build + push

Signed-off-by: Chris Evich <[email protected]>
containers · Sep 13, 2021 · d6a52dd · d6a52dd
1 parent 118c39e
commit d6a52dd
Showing 12 changed files with 948 additions and 19 deletions.
diff --git a/.cirrus.yml b/.cirrus.yml
@@ -16,7 +16,7 @@ container:
 
 
 # Execute all unit-tests in the repo
-cirrus-ci/test_task:
+cirrus-ci/unit-test_task:
     env:
         CIRRUS_CLONE_DEPTH: 0
     script:
@@ -26,11 +26,58 @@ cirrus-ci/test_task:
         test_output_artifacts:
             path: '*.log'
 
+# This is the same setup as used for Buildah CI
+gcp_credentials: ENCRYPTED[fc95bcc9f4506a3b0d05537b53b182e104d4d3979eedbf41cf54205be6397ca0bce0831d0d47580cf578dae5776548a5]
+
+cirrus-ci/build-push_test_task:
+    depends_on:
+        - cirrus-ci/unit-test
+    gce_instance:
+        cpu: 2
+        memory: "4Gb"
+        disk: 200  # Gigabytes, do not set less as per gcloud warning message
+                   # re: I/O performance
+        # This repo. is subsequently used in and for building custom VM images
+        # in containers/automation_images.  Avoid circular dependencies by using
+        # only stock, google-managed generic image.  This also avoids needing to
+        # update custom-image last-used timestamps.
+        image_project: centos-cloud
+        image_family: centos-stream-8
+    timeout_in: 20
+    env:
+        # Need buildah version 1.23 or later (unreleased as of this commit)
+        # TODO: Use a version-tagged image when avail for CI-stability.
+        CIMG: quay.io/buildah/upstream:latest
+        TEST_FQIN: quay.io/buildah/do_not_use
+        # Robot account credentials for test-push to
+        # $TEST_FQIN registry by build-push/test/testbuilds.sh
+        BUILDAH_USERNAME: ENCRYPTED[53fd8becb599dda19f335d65cb067c46da3f0907eb83281a10554def11efc89925f7ca145ba7436afc3c32d936575142]
+        BUILDAH_PASSWORD: ENCRYPTED[aa6352251eba46e389e4cfc6e93eee3852008ecff67b940cba9197fd8bf95de15d498a6df2e7d5edef052e97d9b93bf0]
+    setup_script:
+        - dnf install -y podman
+        - bash build-push/test/qemusetup.sh
+        - >-
+            podman run --detach --name=buildah
+            --net=host --ipc=host --pid=host
+            --cgroupns=host --privileged
+            --security-opt label=disable
+            --security-opt seccomp=unconfined
+            --device /dev/fuse:rw
+            -v $PWD:$PWD:Z -w $PWD
+            -e BUILD_PUSH_TEST_BUILDS=true
+            -e CIRRUS_CI -e TEST_FQIN
+            -e BUILDAH_USERNAME -e BUILDAH_PASSWORD
+            $CIMG
+            sh -c 'while true ;do sleep 2h ; done'
+        - podman exec -i buildah dnf install -y jq skopeo
+    test_script:
+        - podman exec -i buildah ./build-push/test/run_all_tests.sh
+
 
 # Represent primary Cirrus-CI based testing (Required for merge)
 cirrus-ci/success_task:
     depends_on:
-        - cirrus-ci/test
+        - cirrus-ci/build-push_test
     clone_script: mkdir -p "$CIRRUS_WORKING_DIR"
     script: >-
         echo "Required for Action Workflow: https://github.com/${CIRRUS_REPO_FULL_NAME}/actions/runs/${GITHUB_CHECK_SUITE_ID}"
@@ -51,7 +98,7 @@ github-actions/success_task:
     # Only required for PRs, never tag or branch testing
     only_if: $CIRRUS_PR =~ '.*[0-9]+.*'
     depends_on:
-        - cirrus-ci/test
+        - cirrus-ci/build-push_test
     clone_script: mkdir -p "$CIRRUS_WORKING_DIR"
     script: >-
         echo "Triggered by Github Action Workflow: https://github.com/${CIRRUS_REPO_FULL_NAME}/actions/runs/${GITHUB_CHECK_SUITE_ID}"
diff --git a/README.md b/README.md
@@ -18,36 +18,55 @@ During build of an environment (VM, container image, etc), execute *any version*
 of [the install
 script](https://github.com/containers/automation/releases/download/latest/install_automation.sh),
 preferably as root.  The script ***must*** be passed the version number of [the project
-release to install](https://github.com/containers/automation/releases).  Before making
-changes to the environment, the script will first download and then re-execute
-the requested version of itself.
+release to install](https://github.com/containers/automation/releases).  Alternatively
+it may be passed `latest` to install the HEAD of the main branch.
 
 For example, to install the `v1.0.0` release, run:
 ```sh
-url='https://github.com/containers/automation/releases/latest/download/install_automation.sh'
-curl -sL "$url" | bash -s 1.1.3
+# url='https://github.com/containers/automation/releases/latest/download/install_automation.sh'
+# curl -sL "$url" | bash -s 1.1.3
+```
+
+To install `latest`, run:
+```sh
+# url='https://github.com/containers/automation/releases/latest/download/install_automation.sh'
+# curl -sL "$url" | bash -s latest
+```
+
+Optionally, the installer may also be passed the names of one or more components to
+install system-wide.  For example:
+
+To install the latest `build-push`, run:
+```sh
+# url='https://github.com/containers/automation/releases/latest/download/install_automation.sh'
+# curl -sL "$url" | bash -s latest build-push
 ```
 
 ## Alt. Installation
 
-If a clone of the repository is already available locally, the installer can be invoked
-with the magic version number '0.0.0'.  Note that, while it will install the files
-from the local clone as-is, the installer still needs to reach out to github to
-retrieve tree-history details.  This is required for the installer to properly
-set the actual version-number as part of the process.
+If you're leery of piping to bash and/or a local clone of the repository is already
+available locally, the installer can be invoked with the *magic version* '0.0.0'.
+Note that this will limit the install to the local clone (as-is), the installer script
+will still reach out to github.com to retrieve version information.  For example:
 
-Though not recommended at all, it is also possible to specify the version as
-`latest`.  This will clone down whatever happens to be on the 'main' branch
-at the time.  Though it will probably work, it's best for stability to specify
-an explicit released version.
+```sh
+# cd /path/to/clone
+# ./bin/install_automation.sh 0.0.0
+```
 
 ## Usage
 
 The basic install consists of copying the contents of the `common` (subdirectory) and
 the installer script into a central location on the system.  Because this location
 can vary, a global shell variable `$AUTOMATION_LIB_PATH` is widely used.  Therefore,
-it is highly recommended that all users and calling scripts explicitly load
-env. var.  definitions set in the file `/etc/automation_environment`.
+it is highly recommended that all users and calling scripts explicitly load and export
+env. var.  definitions set in the file `/etc/automation_environment`.  For example:
+
+```sh
+# set -a
+# source /etc/automation_environment
+# set +a
+```
 
 
 ## Subdirectories
@@ -71,3 +90,9 @@ other repositories.
 ### `cirrus-ci_retrospective`
 
 See the [README.md file in the subdirectory](cirrus-ci_retrospective/README.md) for more information
+
+### `build-push`
+
+Handy automation too to help with parallel building and pushing container images,
+including support for multi-arch (via QEMU emulation).  See the
+[README.md file in the subdirectory](build-push/README.md) for more information.
diff --git a/build-push/.install.sh b/build-push/.install.sh
@@ -0,0 +1,26 @@
+
+# Installs 'build-push' script system-wide.  NOT intended to be used directly
+# by humans, should only be used indirectly by running
+# ../bin/install_automation.sh <ver> build-push
+
+source "$AUTOMATION_LIB_PATH/anchors.sh"
+source "$AUTOMATION_LIB_PATH/console_output.sh"
+
+INSTALL_PREFIX=$(realpath $AUTOMATION_LIB_PATH/..)
+# Assume the directory this script is in, represents what is being installed
+INSTALL_NAME=$(basename $(dirname ${BASH_SOURCE[0]}))
+AUTOMATION_VERSION=$(automation_version)
+[[ -n "$AUTOMATION_VERSION" ]] || \
+    die "Could not determine version of common automation libs, was 'install_automation.sh' successful?"
+
+echo "Installing $INSTALL_NAME version $(automation_version) into $INSTALL_PREFIX"
+
+unset INST_PERM_ARG
+if [[ $UID -eq 0 ]]; then
+    INST_PERM_ARG="-o root -g root"
+fi
+
+cd $(dirname $(realpath "${BASH_SOURCE[0]}"))
+install -v $INST_PERM_ARG -D -t "$INSTALL_PREFIX/bin" ./bin/*
+
+echo "Successfully installed $INSTALL_NAME"
diff --git a/build-push/README.md b/build-push/README.md
@@ -0,0 +1,113 @@
+# Build-push script
+
+This is a wrapper around buildah build, coupled with pre and post
+build commands and automatic registry server push.  Its goal is to
+provide an abstraction layer for additional build automation. Though
+it may be useful on its own, this is not its primary purpose.
+
+
+## Requirements
+
+* Executables for `jq`, and `buildah` (1.23 or later) are available.
+* Automation common-library is installed & env. var set.
+  * Installed system-wide as per
+    [the top-level documentation](https://github.com/containers/automation#installation)
+  * -or-
+  * Run directly from repository clone by first doing
+    `export AUTOMATION_LIB_PATH=/path/to/clone/common/lib`
+* Optionally, the kernel may be configured to use emulation (such as QEMU)
+  for non-native binary execution (where available and supported).  See
+  [the section below for more
+  infomration](README.md#qemu-user-static-emulation).
+
+
+## QEMU-user-static Emulation
+
+On platforms/distro's that support it, this is a handy way to enable
+non-native binary execution.  This can therefore be used to support
+building container images for other architectures.  Though other
+techniques may be possible, there are [handy/dandy scripts and
+container images available to help configure
+this.](https://github.com/multiarch/qemu-user-static#multiarchqemu-user-static-images)
+
+Using this in a reliable way under automation suggests pointing it
+at the host's qemu-user-staic binaries instead of those bundled in
+the container image (which may not even match the host platform).
+Fortunately passing those in via volume-mount is pretty trivial.
+Something like this (as **root** in Fedora):
+
+```bash
+$ sudo dnf install -y qemu-user-static
+$ qemu_setup_fqin="docker.io/multiarch/qemu-user-static:latest"
+$ vol_awk='{print "-v "$1":"$1""}'
+$ bin_vols=$(find /usr/bin -name 'qemu-*-static' | awk -e "$vol_awk" | tr '\n' ' ')
+$ sudo podman run --rm --privileged $bin_vols $qemu_setup_fqin --reset -p yes
+```
+
+Note: You may need to alter `$vol_awk` or the `podman` command line
+depending on what your platform supports.
+
+
+## Use in build automation
+
+This script may be useful as a uniform interface for building and pushing
+for multiple architectures, all in one go.  A simple example would be:
+
+```bash
+$ export SOME_USERNAME=foo  # normally hidden/secured in the CI system
+$ export SOME_PASSWORD=bar  # along with this password value.
+
+$ build-push.sh --arches=arm64,ppc64le,s390x quay.io/some/thing ./path/to/contextdir
+```
+
+In this case, the image `quay.io/some/thing:latest` would be built for the
+listed architectures, then pushed to the remote registry server.
+
+### Use in automation with additional preparation
+
+When building for multiple architectures using emulation, it's vastly
+more efficient to execute as few non-native RUN instructions as possible.
+This is supported by the `--prepcmd` option, which specifies a shell
+command-string to execute prior to building the image. The command-string
+will have access to a set of exported env. vars. for use and/or
+substitution (see the `--help` output for details).
+
+For example, this command string could be used to seed the build cache
+by pulling down previously built image of the same name:
+
+```bash
+$ build-push.sh ... quay.io/test/ing --prepcmd='$RUNTIME pull $FQIN:latest'
+```
+
+In this example, the command `buildah pull quay.io/test/ing:latest` will
+be executed prior to the build.
+
+### Use in automation with modified images
+
+Sometimes additional steps need to be performed after the build, to modify,
+inspect or additionally tag the built image before it's pushed.  This could
+include (for example) running tests on the image, or modifying its metadata
+in some way.  All these and more are supported by the `--modcmd` option.
+
+Simply feed it a command string to be run after a successful build.  The
+command-string script will have access to a set of exported env. vars.
+for use and/or substitution (see the `--help` output for details).
+
+After executing a `--modcmd`, `build-push.sh` will take care to identify
+all images related to the original FQIN (minus the tag).  Should
+additional tags be present, they will also be pushed (absent the
+`--nopush` flag).  If any/all images are missing, they will be silently
+ignored.
+
+For example you could use this to only push version-tagged images, and
+never `latest`:
+
+```
+$ build-push.sh ... --modcmd='$RUNTIME tag $FQIN:latest $FQIN:9.8.7 && \
+                              $RUNTIME manifest rm $FQIN:latest'
+```
+
+Note: If your `--modcmd` command or script removes **ALL** tags, and
+`--nopush` was **not** specified, an error message will be printed
+followed by a non-zero exit.  This is intended to help automation
+catch an assumed missed-expectation.
diff --git a/build-push/bin/build-push.sh b/build-push/bin/build-push.sh
diff --git a/build-push/test/fake_buildah.sh b/build-push/test/fake_buildah.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "$@"
diff --git a/build-push/test/qemusetup.sh b/build-push/test/qemusetup.sh
@@ -0,0 +1,30 @@
+
+
+# This script is intend for use by tests, DO NOT EXECUTE.
+
+set -eo pipefail
+
+if  [[ "$CIRRUS_CI" == "true" ]]; then
+    # Cirrus-CI is setup (see .cirrus.yml) to run tests on CentOS
+    # for simplicity, but it has no native qemu-user-static.  For
+    # the benefit of CI testing, cheat and use whatever random
+    # emulators are included in the container image.
+
+    # Workaround silly stupid hub rate-limiting
+    cat >> /etc/containers/registries.conf << EOF
+[[registry]]
+prefix="docker.io/library"
+location="mirror.gcr.io"
+EOF
+
+    # N/B: THIS IS NOT SAFE FOR PRODUCTION USE!!!!!
+    podman run --rm --privileged \
+        docker.io/multiarch/qemu-user-static:latest \
+        --reset -p yes
+elif [[ -x "/usr/bin/qemu-aarch64-static" ]]; then
+    # TODO: Better way to determine if kernel already setup?
+    echo "Warning: Assuming qemu-user-static is already setup"
+else
+    echo "Error: System does not appear to have qemu-user-static setup"
+    exit 1
+fi
diff --git a/build-push/test/run_all_tests.sh b/build-push/test/run_all_tests.sh
@@ -0,0 +1 @@
+../../common/test/run_all_tests.sh
diff --git a/build-push/test/test_context/Containerfile b/build-push/test/test_context/Containerfile
@@ -0,0 +1,4 @@
+FROM registry.fedoraproject.org/fedora-minimal:latest
+RUN /bin/true
+ENTRYPOINT /bin/false
+# WARNING: testbuilds.sh depends on the number of build steps
diff --git a/build-push/test/testbin-build-push.sh b/build-push/test/testbin-build-push.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+
+TEST_SOURCE_DIRPATH=$(realpath $(dirname "${BASH_SOURCE[0]}"))
+
+# Load standardized test harness
+source $TEST_SOURCE_DIRPATH/testlib.sh || exit 1
+
+SUBJ_FILEPATH="$TEST_DIR/$SUBJ_FILENAME"
+TEST_CONTEXT="$TEST_SOURCE_DIRPATH/test_context"
+EMPTY_CONTEXT=$(mktemp -d -p '' .tmp_$(basename ${BASH_SOURCE[0]})_XXXX)
+
+test_cmd "Verify error when automation library not found" \
+    1 'ERROR: Expecting \$AUTOMATION_LIB_PATH to be set' \
+    bash -c "AUTOMATION_LIB_PATH='' RUNTIME=/bin/true $SUBJ_FILEPATH 2>&1"
+
+export AUTOMATION_LIB_PATH="$TEST_SOURCE_DIRPATH/../../common/lib"
+
+test_cmd "Verify error when buildah can't be found" \
+    1 "ERROR:.+find buildah.+/usr/local/bin" \
+    bash -c "RUNTIME=/bin/true $SUBJ_FILEPATH 2>&1"
+
+# Support basic testing w/o a buildah binary available
+export RUNTIME="${RUNTIME:-$(type -P buildah)}"
+export NATIVE_GOARCH="${NATIVE_GOARCH:-$($RUNTIME info --format='{{.host.arch}}')}"
+export PARALLEL_JOBS="${PARALLEL_JOBS:-$($RUNTIME info --format='{{.host.cpus}}')}"
+
+# These tests don't actually need to actually build/run anything
+export OLD_RUNTIME="$RUNTIME"
+export RUNTIME="$TEST_SOURCE_DIRPATH/fake_buildah.sh"
+
+test_cmd "Verify error when executed w/o any arguments" \
+    1 "ERROR: Must.+required arguments." \
+    bash -c "$SUBJ_FILEPATH 2>&1"
+
+test_cmd "Verify error when specify partial required arguments" \
+    1 "ERROR: Must.+required arguments." \
+    bash -c "$SUBJ_FILEPATH foo 2>&1"
+
+test_cmd "Verify error when executed bad Containerfile directory" \
+    1 "ERROR:.+directory: 'bar'" \
+    bash -c "$SUBJ_FILEPATH foo bar 2>&1"
+
+test_cmd "Verify error when specify invalid FQIN" \
+    1 "ERROR:.+FQIN.+foo" \
+    bash -c "$SUBJ_FILEPATH foo $EMPTY_CONTEXT 2>&1"
+
+test_cmd "Verify error when specify slightly invalid FQIN" \
+    1 "ERROR:.+FQIN.+foo/bar" \
+    bash -c "$SUBJ_FILEPATH foo/bar $EMPTY_CONTEXT 2>&1"
+
+test_cmd "Verify error when executed bad context subdirectory" \
+    1 "ERROR:.+Containerfile or Dockerfile: '$EMPTY_CONTEXT'" \
+    bash -c "$SUBJ_FILEPATH foo/bar/baz $EMPTY_CONTEXT 2>&1"
+
+# no-longer needed
+rm -rf "$EMPTY_CONTEXT"
+unset EMPTY_CONTEXT
+
+test_cmd "Verify --help output to stdout can be grepped" \
+    0 "Optional Environment Variables:" \
+    bash -c "$SUBJ_FILEPATH --help | grep 'Optional Environment Variables:'"
+
+test_cmd "Confirm required username env. var. unset error" \
+    1 "ERROR.+BAR_USERNAME" \
+    bash -c "$SUBJ_FILEPATH foo/bar/baz $TEST_CONTEXT 2>&1"
+
+test_cmd "Confirm required password env. var. unset error" \
+    1 "ERROR.+BAR_PASSWORD" \
+    bash -c "BAR_USERNAME=snafu $SUBJ_FILEPATH foo/bar/baz $TEST_CONTEXT 2>&1"
+
+for arg in 'prepcmd' 'modcmd'; do
+    test_cmd "Verify error when --$arg specified without an '='" \
+        1 "ERROR:.+with '='" \
+        bash -c "BAR_USERNAME=snafu BAR_PASSWORD=ufans $SUBJ_FILEPATH foo/bar/baz $TEST_CONTEXT --$arg notgoingtowork 2>&1"
+done
+
+# A specialized non-container environment required to run these
+if [[ -n "$BUILD_PUSH_TEST_BUILDS" ]]; then
+    unset RUNTIME NATIVE_GOARCH PARALLEL_JOBS
+    RUNTIME="$OLD_RUNTIME"
+    export RUNTIME
+
+    source $(dirname "${BASH_SOURCE[0]}")/testbuilds.sh
+else
+    echo "WARNING: Set \$BUILD_PUSH_TEST_BUILDS non-empty to fully test build_push."
+    echo ""
+fi
+
+# Must always happen last
+exit_with_status
diff --git a/build-push/test/testbuilds.sh b/build-push/test/testbuilds.sh
@@ -0,0 +1,143 @@
+
+
+# This script is intended to be sourced from testbin-build-push.sh.
+# Any/all other usage is virtually guaranteed to fail and/or cause
+# harm to the system.
+
+for varname in RUNTIME TEST_FQIN BUILDAH_USERNAME BUILDAH_PASSWORD; do
+    value=${!varname}
+    if [[ -z "$value" ]]; then
+        echo "ERROR: Required \$$varname variable is unset/empty."
+        exit 1
+    fi
+done
+unset value
+
+$RUNTIME --version
+test_cmd "Confirm $(basename $RUNTIME) is available" \
+    0 "buildah version .+" \
+    $RUNTIME --version
+
+skopeo --version
+test_cmd "Confirm skopeo is available" \
+    0 "skopeo version .+" \
+    skopeo --version
+
+PREPCMD='echo "SpecialErrorMessage:$REGSERVER" > /dev/stderr && exit 42'
+test_cmd "Confirm error output and exit(42) from --prepcmd" \
+    42 "SpecialErrorMessage:localhost" \
+    bash -c "$SUBJ_FILEPATH --nopush localhost/foo/bar $TEST_CONTEXT --prepcmd='$PREPCMD' 2>&1"
+
+# N/B: The following are stateful - each depends on precedding test success
+#      and assume empty container-storage (podman system reset).
+
+test_cmd "Confirm building native-arch test image w/ --nopush" \
+    0 "STEP 3/3: ENTRYPOINT /bin/false.+COMMIT" \
+    bash -c "DEBUG=1 $SUBJ_FILEPATH localhost/foo/bar $TEST_CONTEXT --nopush 2>&1"
+
+native_arch=$($RUNTIME info --format='{{.host.arch}}')
+test_cmd "Confirm native_arch was set to non-empty string" \
+    0 "" \
+    test -n "$native_arch"
+
+test_cmd "Confirm built image manifest contains the native arch '$native_arch'" \
+    0 "$native_arch" \
+    bash -c "$RUNTIME manifest inspect localhost/foo/bar:latest | jq -r '.manifests[0].platform.architecture'"
+
+test_cmd "Confirm rebuilding with same command uses cache" \
+    0 "STEP 3/3.+Using cache" \
+    bash -c "DEBUG=1 $SUBJ_FILEPATH localhost/foo/bar $TEST_CONTEXT --nopush 2>&1"
+
+test_cmd "Confirm manifest-list can be removed by name" \
+    0 "untagged: localhost/foo/bar:latest" \
+    $RUNTIME manifest rm containers-storage:localhost/foo/bar:latest
+
+test_cmd "Verify expected partial failure when passing bogus architectures" \
+     125 "error creating build.+architecture staple" \
+    bash -c "DEBUG=1 $SUBJ_FILEPATH --arches=correct,horse,battery,staple localhost/foo/bar --nopush $TEST_CONTEXT 2>&1"
+
+MODCMD='$RUNTIME tag $FQIN:latest $FQIN:9.8.7-testing'
+test_cmd "Verify --modcmd is able to tag the manifest" \
+    0 "Executing mod-command" \
+    bash -c "DEBUG=1 $SUBJ_FILEPATH localhost/foo/bar $TEST_CONTEXT --nopush --modcmd='$MODCMD' 2>&1"
+
+test_cmd "Verify the tagged manifest is also present" \
+    0 "[a-zA-Z0-9]+" \
+    bash -c "$RUNTIME images --quiet localhost/foo/bar:9.8.7-testing"
+
+test_cmd "Confirm tagged image manifest contains native arch '$native_arch'" \
+    0 "$native_arch" \
+    bash -c "$RUNTIME manifest inspect localhost/foo/bar:9.8.7-testing | jq -r '.manifests[0].platform.architecture'"
+
+TEST_TEMP=$(mktemp -d -p '' .tmp_$(basename ${BASH_SOURCE[0]})_XXXX)
+
+test_cmd "Confirm digest can be obtained from 'latest' manifest list" \
+    0 ".+" \
+    bash -c "$RUNTIME manifest inspect locahost/foo/bar:latest | jq -r '.manifest[0].digest' | tee $TEST_TEMP/latest_digest"
+
+test_cmd "Confirm digest can be obtained from '9.8.7-testing' manifest list" \
+    0 ".+" \
+    bash -c "$RUNTIME manifest inspect locahost/foo/bar:9.8.7-testing | jq -r '.manifest[0].digest' | tee $TEST_TEMP/tagged_digest"
+
+test_cmd "Verify tagged manifest image digest matches the same in latest" \
+    0 "" \
+    test "$(<$TEST_TEMP/tagged_digest)" == "$(<$TEST_TEMP/latest_digest)"
+
+MODCMD='
+set -x;
+$RUNTIME images && \
+    $RUNTIME manifest rm containers-storage:$FQIN:latest && \
+    $RUNTIME manifest rm containers-storage:$FQIN:9.8.7-testing && \
+    echo "AllGone";
+'
+# TODO: Test fails due to: https://github.com/containers/buildah/issues/3490
+# for now pretend it should exit(125) which will be caught when bug is fixed
+# - causing it to exit(0) as it should
+test_cmd "Verify --modcmd can execute a long string with substitutions" \
+    125 "AllGone" \
+    bash -c "DEBUG=1 $SUBJ_FILEPATH --modcmd='$MODCMD' localhost/foo/bar --nopush $TEST_CONTEXT 2>&1"
+
+test_cmd "Verify previous --modcmd removed the 'latest' tagged image" \
+    125 "image not known" \
+    $RUNTIME images --quiet containers-storage:localhost/foo/bar:latest
+
+test_cmd "Verify previous --modcmd removed the '9.8.7-testing' tagged image" \
+    125 "image not known" \
+    $RUNTIME images --quiet containers-storage:localhost/foo/bar:9.8.7-testing
+
+FAKE_VERSION=$RANDOM
+MODCMD="set -ex;
+\$RUNTIME tag \$FQIN:latest \$FQIN:$FAKE_VERSION;
+\$RUNTIME manifest rm \$FQIN:latest;"
+test_cmd "Verify e2e workflow w/ additional build-args" \
+    0 "Pushing $TEST_FQIN:$FAKE_VERSION" \
+    bash -c "env DEBUG=1 $SUBJ_FILEPATH \
+        --prepcmd='touch $TEST_SOURCE_DIRPATH/test_context/Containerfile' \
+        --modcmd='$MODCMD' \
+        --arches=amd64,s390x,arm64,ppc64le \
+        $TEST_FQIN \
+        $TEST_CONTEXT \
+        --device=/dev/fuse --label testing=true \
+        2>&1"
+
+test_cmd "Verify latest tagged image was not pushed" \
+    1 "(Tag latest was deleted or has expired.)|(manifest unknown: manifest unknown)" \
+    skopeo inspect docker://$TEST_FQIN:latest
+
+test_cmd "Verify architectures can be obtained from manifest list" \
+    0 "" \
+    bash -c "$RUNTIME manifest inspect $TEST_FQIN:$FAKE_VERSION | \
+        jq -r '.manifests[].platform.architecture' > $TEST_TEMP/maniarches"
+
+for arch in amd64 s390x arm64 ppc64le; do
+    test_cmd "Verify $arch architecture present in $TEST_FQIN:$FAKE_VERSION" \
+    0 "" \
+    fgrep -qx "$arch" $TEST_TEMP/maniarches
+done
+
+test_cmd "Verify pushed image can be removed" \
+    0 "" \
+    skopeo delete docker://$TEST_FQIN:$FAKE_VERSION
+
+# Cleanup
+rm -rf "$TEST_TEMP"
diff --git a/build-push/test/testlib.sh b/build-push/test/testlib.sh
@@ -0,0 +1 @@
+../../common/test/testlib.sh