diff --git a/.cirrus.yml b/.cirrus.yml index 7ba243c5d8..56e2ff8887 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -413,7 +413,8 @@ bindings_task: swagger_task: name: "Test Swagger" alias: swagger - depends_on: *build + depends_on: + - buildah_bud_test gce_instance: *standardvm env: <<: *stdenvars @@ -441,7 +442,8 @@ win_installer_task: CONTAINERS_MACHINE_PROVIDER: 'hyperv' alias: win_installer only_if: *no_rhel_release - depends_on: *build + depends_on: + - buildah_bud_test ec2_instance: &windows image: "${WINDOWS_AMI}" type: m5.large @@ -509,7 +511,8 @@ docker-py_test_task: changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('test/python/**') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test gce_instance: *standardvm env: <<: *stdenvars @@ -536,7 +539,8 @@ unit_test_task: changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('**/*_test.go') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test matrix: - env: *stdenvars # Special-case: Rootless on latest Fedora (standard) VM @@ -566,7 +570,8 @@ apiv2_test_task: changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('test/apiv2/**', 'test/python/**') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test gce_instance: *standardvm env: <<: *stdenvars @@ -595,7 +600,8 @@ compose_test_task: changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('test/compose/**') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test gce_instance: *standardvm matrix: - env: @@ -627,7 +633,8 @@ local_integration_test_task: &local_integration_test_task changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('test/e2e/**', 'test/utils/**') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test matrix: *platform_axis # integration tests scale well with cpu as they are parallelized # so we give these tests 4 cores to make them faster @@ -662,7 +669,8 @@ container_integration_test_task: alias: container_integration_test # Docs: ./contrib/cirrus/CIModes.md only_if: *only_if_int_test - depends_on: *build + depends_on: + - buildah_bud_test matrix: &fedora_vm_axis - env: DISTRO_NV: ${FEDORA_NAME} @@ -689,7 +697,8 @@ rootless_integration_test_task: alias: rootless_integration_test # Docs: ./contrib/cirrus/CIModes.md only_if: *only_if_int_test - depends_on: *build + depends_on: + - buildah_bud_test matrix: *platform_axis gce_instance: *fastvm env: @@ -712,7 +721,8 @@ podman_machine_task: $CIRRUS_CHANGE_TITLE =~ '.*CI:ALL.*' || changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('cmd/podman/machine/**', 'pkg/machine/**', '**/*machine*.go') - depends_on: *build + depends_on: + - buildah_bud_test ec2_instance: image: "${VM_IMAGE_NAME}" type: "${EC2_INST_TYPE}" @@ -734,7 +744,8 @@ podman_machine_aarch64_task: name: *std_name_fmt alias: podman_machine_aarch64 only_if: *only_if_machine_test - depends_on: *build + depends_on: + - buildah_bud_test ec2_instance: <<: *standard_build_ec2_aarch64 timeout_in: 30m @@ -767,7 +778,8 @@ podman_machine_windows_task: skip: &skip_rhel_release | $CIRRUS_BRANCH =~ 'v[0-9\.]+-rhel' || $CIRRUS_BASE_BRANCH =~ 'v[0-9\.]+-rhel' - depends_on: *build + depends_on: + - buildah_bud_test ec2_instance: <<: *windows type: m5zn.metal @@ -793,7 +805,8 @@ podman_machine_mac_task: alias: podman_machine_mac only_if: *only_if_machine_test skip: *skip_rhel_release - depends_on: *build + depends_on: + - buildah_bud_test persistent_worker: *mac_pw timeout_in: 35m env: @@ -849,7 +862,8 @@ local_system_test_task: &local_system_test_task changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('test/system/**') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test matrix: *platform_axis gce_instance: *fastvm timeout_in: 25m @@ -866,7 +880,8 @@ local_system_test_aarch64_task: &local_system_test_task_aarch64 alias: local_system_test_aarch64 # Docs: ./contrib/cirrus/CIModes.md only_if: *only_if_system_test - depends_on: *build + depends_on: + - buildah_bud_test ec2_instance: *standard_build_ec2_aarch64 timeout_in: 35m env: @@ -918,7 +933,8 @@ rootless_system_test_task: alias: rootless_system_test # Docs: ./contrib/cirrus/CIModes.md only_if: *only_if_system_test - depends_on: *build + depends_on: + - buildah_bud_test matrix: *platform_axis gce_instance: *fastvm timeout_in: 25m @@ -944,7 +960,8 @@ farm_test_task: changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('test/farm/**', 'test/system/*.bash') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test gce_instance: *standardvm env: <<: *stdenvars @@ -976,7 +993,13 @@ buildah_bud_test_task: PODBIN_NAME: podman - env: PODBIN_NAME: remote - gce_instance: *standardvm + - env: + PODBIN_NAME: podman + PRIV_NAME: rootless + - env: + PODBIN_NAME: remote + PRIV_NAME: rootless + gce_instance: *fastvm timeout_in: 45m clone_script: *get_gosrc setup_script: *setup @@ -996,7 +1019,8 @@ upgrade_test_task: changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') || changesInclude('test/upgrade/**', 'test/system/*.bash') || (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**')) - depends_on: *build + depends_on: + - buildah_bud_test matrix: - env: # 2024-02: as long as possible/reasonable, try to keep diff --git a/go.mod b/go.mod index 87688ae12c..f6cb18f9f7 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/checkpoint-restore/checkpointctl v1.3.0 github.com/checkpoint-restore/go-criu/v7 v7.2.0 github.com/containernetworking/plugins v1.5.1 - github.com/containers/buildah v1.38.1-0.20241119213149-52437ef15d33 + github.com/containers/buildah v1.38.1-0.20241122025341-998312277d88 github.com/containers/common v0.61.1-0.20241112152446-305e9ce69b0f github.com/containers/conmon v2.0.20+incompatible github.com/containers/gvisor-tap-vsock v0.8.0 diff --git a/go.sum b/go.sum index 5c18990206..3c8f1111ab 100644 --- a/go.sum +++ b/go.sum @@ -79,8 +79,8 @@ github.com/containernetworking/cni v1.2.3 h1:hhOcjNVUQTnzdRJ6alC5XF+wd9mfGIUaj8F github.com/containernetworking/cni v1.2.3/go.mod h1:DuLgF+aPd3DzcTQTtp/Nvl1Kim23oFKdm2okJzBQA5M= github.com/containernetworking/plugins v1.5.1 h1:T5ji+LPYjjgW0QM+KyrigZbLsZ8jaX+E5J/EcKOE4gQ= github.com/containernetworking/plugins v1.5.1/go.mod h1:MIQfgMayGuHYs0XdNudf31cLLAC+i242hNm6KuDGqCM= -github.com/containers/buildah v1.38.1-0.20241119213149-52437ef15d33 h1:Ih6KuyByK7ZGGzkS0M5rVBPLWIyeDvdL5klhsKBo8vA= -github.com/containers/buildah v1.38.1-0.20241119213149-52437ef15d33/go.mod h1:RxIuKhwTpRl3ma4d4BF6QzSSeg9zNNvo/xhYJOKeDQs= +github.com/containers/buildah v1.38.1-0.20241122025341-998312277d88 h1:Xa1sfUgLrIuXx5q3JtyQmg/6suD/lWu0PqcdGFQFrSU= +github.com/containers/buildah v1.38.1-0.20241122025341-998312277d88/go.mod h1:RxIuKhwTpRl3ma4d4BF6QzSSeg9zNNvo/xhYJOKeDQs= github.com/containers/common v0.61.1-0.20241112152446-305e9ce69b0f h1:K3jmJrkDJJhLnRdVFI7Gb5mv4/jb2ue9StZ2F1y2rsE= github.com/containers/common v0.61.1-0.20241112152446-305e9ce69b0f/go.mod h1:NGRISq2vTFPSbhNqj6MLwyes4tWSlCnqbJg7R77B8xc= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= diff --git a/test/buildah-bud/buildah-tests.diff b/test/buildah-bud/buildah-tests.diff index 039d651c94..2f9a60f1e9 100644 --- a/test/buildah-bud/buildah-tests.diff +++ b/test/buildah-bud/buildah-tests.diff @@ -1,21 +1,22 @@ -From d7839d93860915b2a43c486d0fed89fee7313ec0 Mon Sep 17 00:00:00 2001 +From 42a2977ef907112fd48575bcf69aef7847726a9b Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Thu, 6 Oct 2022 17:32:59 -0600 Subject: [PATCH] tweaks for running buildah tests under podman Signed-off-by: Ed Santiago --- - tests/helpers.bash | 119 +++++++++++++++++++++++++++++++++++++++++++-- - 1 file changed, 115 insertions(+), 4 deletions(-) + tests/helpers.bash | 126 +++++++++++++++++++++++++++++++++++++++++++-- + 1 file changed, 122 insertions(+), 4 deletions(-) diff --git a/tests/helpers.bash b/tests/helpers.bash -index f8ab624a8..0d8f5ce69 100644 +index ed5de994e..8f82a9fd4 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash -@@ -80,6 +80,38 @@ EOF +@@ -80,6 +80,42 @@ EOF BUILDAH_REGISTRY_OPTS="${regconfopt} ${regconfdir} --short-name-alias-conf ${TEST_SCRATCH_DIR}/cache/shortnames.conf" COPY_REGISTRY_OPTS="${BUILDAH_REGISTRY_OPTS}" PODMAN_REGISTRY_OPTS="${regconfopt}" ++ PODMAN_REMOTE_OPTS= + + PODMAN_SERVER_PID= + PODMAN_NATIVE="${PODMAN_BINARY} ${ROOTDIR_OPTS} ${PODMAN_REGISTRY_OPTS}" @@ -32,26 +33,29 @@ index f8ab624a8..0d8f5ce69 100644 + local sockdir=/run + if is_rootless; then + sockdir=${XDG_RUNTIME_DIR:-/run/user/$(id -u)} ++ mkdir -p ${sockdir}/podman + fi -+ PODMAN_SOCK_FILE=$sockdir/podman/podman.sock ++ PODMAN_SOCK_FILE=$sockdir/podman/podman-${BATS_SUITE_TEST_NUMBER}.sock ++ PODMAN_REMOTE_OPTS="--url unix://${PODMAN_SOCK_FILE}" + # static CONTAINERS_CONF needed for capabilities test. As of 2021-07-01 + # no tests in bud.bats override this; if at some point any test does + # so, it will probably need to be skip_if_remote()d. -+ env CONTAINERS_CONF_OVERRIDE=${CONTAINERS_CONF_OVERRIDE:-$(dirname ${BASH_SOURCE})/containers.conf} $PODMAN_NATIVE system service --log-level=info --timeout=0 &>>${PODMAN_SERVER_LOG:-/dev/stderr} & ++ echo "$_LOG_PROMPT $PODMAN_NATIVE system service [...] unix://${PODMAN_SOCK_FILE}" >&2 ++ env CONTAINERS_CONF_OVERRIDE=${CONTAINERS_CONF_OVERRIDE:-$(dirname ${BASH_SOURCE})/containers.conf} $PODMAN_NATIVE system service --log-level=info --timeout=0 unix://${PODMAN_SOCK_FILE} &>>${PODMAN_SERVER_LOG:-/dev/stderr} & + PODMAN_SERVER_PID=$! + echo ">> pid=$PODMAN_SERVER_PID" >>${PODMAN_SERVER_LOG:-/dev/stderr} -+ local timeout=10 ++ local timeout=30 + while ((timeout > 0)); do + test -S $PODMAN_SOCK_FILE && return + sleep 0.2 + timeout=$((timeout - 1)) + done -+ die "podman server never came up" ++ die "podman server never came up: $PODMAN_SOCK_FILE" + fi } function starthttpd() { # directory [working-directory-or-"" [certfile, keyfile]] -@@ -144,6 +176,32 @@ function teardown_tests() { +@@ -144,6 +180,35 @@ function teardown_tests() { stop_git_daemon stop_registry @@ -71,20 +75,23 @@ index f8ab624a8..0d8f5ce69 100644 + done + fi + ++ # FIXME! 2024-11-22: I have no idea if this is still relevant, but it ++ # certainly does not play well with parallel bats tests. Let's see ++ # what happens if we disable it. + # FIXME! 2023-04-11: under remote + rootless, on the very first test, + # we somehow end up with two podman-system-service jobs. The second one + # lingers, and prevents BATS from completing, manifesting as a test hang. -+ if is_rootless; then -+ ps auxww | grep "system service" | grep -v grep | while read user pid rest; do -+ echo "# teardown: killing stray server: $user $pid $rest" >&3 -+ kill $pid -+ done -+ fi ++# if is_rootless; then ++# ps auxww | grep "system service" | grep -v grep | while read user pid rest; do ++# echo "# teardown: killing stray server: $user $pid $rest" >&3 ++# kill $pid ++# done ++# fi + # Workaround for #1991 - buildah + overlayfs leaks mount points. # Many tests leave behind /var/tmp/.../root/overlay and sub-mounts; # let's find those and clean them up, otherwise 'rm -rf' fails. -@@ -252,7 +310,12 @@ function copy() { +@@ -265,7 +330,12 @@ function copy() { } function podman() { @@ -92,13 +99,13 @@ index f8ab624a8..0d8f5ce69 100644 + local cmd=${PODMAN_BINARY:-podman} + local opts="${PODMAN_REGISTRY_OPTS} ${ROOTDIR_OPTS}" + if [[ $cmd =~ remote ]]; then -+ opts= ++ opts="${PODMAN_REMOTE_OPTS}" + fi + command $cmd $opts "$@" } # There are various scenarios where we would like to execute `tests` as rootless user, however certain commands like `buildah mount` -@@ -316,8 +379,36 @@ function run_buildah() { +@@ -329,8 +399,36 @@ function run_buildah() { --retry) retry=3; shift;; # retry network flakes esac @@ -117,7 +124,7 @@ index f8ab624a8..0d8f5ce69 100644 + podman_or_buildah=${PODMAN_BINARY} + _opts="${ROOTDIR_OPTS} ${PODMAN_REGISTRY_OPTS}" + if [[ -n "$REMOTE" ]]; then -+ _opts= ++ _opts="${PODMAN_REMOTE_OPTS}" + fi + + # Special case: there's one test that invokes git in such @@ -136,7 +143,7 @@ index f8ab624a8..0d8f5ce69 100644 # If session is rootless and `buildah mount` is invoked, perform unshare, # since normal user cannot mount a filesystem unless they're in a user namespace along with its own mount namespace. -@@ -331,8 +422,8 @@ function run_buildah() { +@@ -344,8 +442,8 @@ function run_buildah() { retry=$(( retry - 1 )) # stdout is only emitted upon error; this echo is to help a debugger @@ -147,7 +154,7 @@ index f8ab624a8..0d8f5ce69 100644 # without "quotes", multiple lines are glommed together into one if [ -n "$output" ]; then echo "$output" -@@ -693,6 +784,26 @@ function skip_if_no_unshare() { +@@ -706,6 +804,26 @@ function skip_if_no_unshare() { fi } diff --git a/test/buildah-bud/run-buildah-bud-tests b/test/buildah-bud/run-buildah-bud-tests index 3407eb8c7c..0ed3a2e3be 100755 --- a/test/buildah-bud/run-buildah-bud-tests +++ b/test/buildah-bud/run-buildah-bud-tests @@ -262,5 +262,5 @@ review the test failure and double-check your changes. BUILDAH_BINARY=$(pwd)/bin/buildah \ COPY_BINARY=$(pwd)/bin/copy \ INET_BINARY=$(pwd)/bin/inet \ - bats "${bats_filter[@]}" tests/bud.bats) + bats -j $(nproc) "${bats_filter[@]}" tests/bud.bats) fi diff --git a/vendor/github.com/containers/buildah/Makefile b/vendor/github.com/containers/buildah/Makefile index 8ca47072da..3d7048ace2 100644 --- a/vendor/github.com/containers/buildah/Makefile +++ b/vendor/github.com/containers/buildah/Makefile @@ -38,7 +38,12 @@ CNI_COMMIT := $(shell sed -n 's;^$(COMMENT) github.com/containernetworking/cni \ EXTRA_LDFLAGS ?= BUILDAH_LDFLAGS := $(GO_LDFLAGS) '-X main.GitCommit=$(GIT_COMMIT) -X main.buildInfo=$(SOURCE_DATE_EPOCH) -X main.cniVersion=$(CNI_COMMIT) $(EXTRA_LDFLAGS)' -SOURCES=*.go imagebuildah/*.go bind/*.go chroot/*.go copier/*.go define/*.go docker/*.go internal/config/*.go internal/mkcw/*.go internal/mkcw/types/*.go internal/parse/*.go internal/sbom/*.go internal/source/*.go internal/tmpdir/*.go internal/*.go internal/util/*.go internal/volumes/*.go manifests/*.go pkg/binfmt/*.go pkg/blobcache/*.go pkg/chrootuser/*.go pkg/cli/*.go pkg/completion/*.go pkg/formats/*.go pkg/jail/*.go pkg/overlay/*.go pkg/parse/*.go pkg/rusage/*.go pkg/sshagent/*.go pkg/umask/*.go pkg/util/*.go pkg/volumes/*.go util/*.go + +# This isn't what we actually build; it's a superset, used for target +# dependencies. Basically: all *.go and *.c files, except *_test.go, +# and except anything in a dot subdirectory. If any of these files is +# newer than our target (bin/buildah), a rebuild is triggered. +SOURCES=$(shell find . -path './.*' -prune -o \( \( -name '*.go' -o -name '*.c' \) -a ! -name '*_test.go' \) -print) LINTFLAGS ?= @@ -68,7 +73,7 @@ static: mkdir -p ./bin cp -rfp ./result/bin/* ./bin/ -bin/buildah: $(SOURCES) cmd/buildah/*.go internal/mkcw/embed/entrypoint_amd64.gz +bin/buildah: $(SOURCES) internal/mkcw/embed/entrypoint_amd64.gz $(GO_BUILD) $(BUILDAH_LDFLAGS) $(GO_GCFLAGS) "$(GOGCFLAGS)" -o $@ $(BUILDFLAGS) ./cmd/buildah test -z "${SELINUXOPT}" || chcon --verbose -t $(SELINUXTYPE) $@ @@ -98,13 +103,13 @@ bin/buildah.%: mkdir -p ./bin GOOS=$(word 2,$(subst ., ,$@)) GOARCH=$(word 3,$(subst ., ,$@)) $(GO_BUILD) $(BUILDAH_LDFLAGS) -o $@ -tags "containers_image_openpgp" ./cmd/buildah -bin/imgtype: $(SOURCES) tests/imgtype/imgtype.go +bin/imgtype: $(SOURCES) $(GO_BUILD) $(BUILDAH_LDFLAGS) -o $@ $(BUILDFLAGS) ./tests/imgtype/imgtype.go -bin/copy: $(SOURCES) tests/copy/copy.go +bin/copy: $(SOURCES) $(GO_BUILD) $(BUILDAH_LDFLAGS) -o $@ $(BUILDFLAGS) ./tests/copy/copy.go -bin/tutorial: $(SOURCES) tests/tutorial/tutorial.go +bin/tutorial: $(SOURCES) $(GO_BUILD) $(BUILDAH_LDFLAGS) -o $@ $(BUILDFLAGS) ./tests/tutorial/tutorial.go bin/inet: tests/inet/inet.go @@ -127,7 +132,6 @@ validate: install.tools ./tests/validate/whitespace.sh ./hack/xref-helpmsgs-manpages ./tests/validate/pr-should-include-tests - ./hack/makefile_sources .PHONY: install.tools install.tools: diff --git a/vendor/modules.txt b/vendor/modules.txt index 6c325120db..b2b17dccd9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -143,7 +143,7 @@ github.com/containernetworking/cni/pkg/version # github.com/containernetworking/plugins v1.5.1 ## explicit; go 1.20 github.com/containernetworking/plugins/pkg/ns -# github.com/containers/buildah v1.38.1-0.20241119213149-52437ef15d33 +# github.com/containers/buildah v1.38.1-0.20241122025341-998312277d88 ## explicit; go 1.22.6 github.com/containers/buildah github.com/containers/buildah/bind