system tests: the catch-up game

- run test: minor cleanup to .containerenv test. Basically,
  make it do only two podman-runs (they're expensive) and
  tighten up the results checks

- ps test: add ps -a --storage. Requires small tweak to
  run_podman helper, so we can have "timeout" be an expected
  result

- sdnotify test: workaround for containers#8718 (seeing MAINPID=xxx as
  last output line instead of READY=1). As found by the
  newly-added debugging echos, what we are seeing is:

      MAINPID=103530
      READY=1
      MAINPID=103530

  It's not supposed to be that way; it's supposed to be just
  the first two. But when faced with reality, we must bend
  to accommodate it, so let's accept READY=1 anywhere in
  the output stream, not just as the last line.

Signed-off-by: Ed Santiago <[email protected]>

test/system/030-run.bats

@@ -548,27 +548,33 @@ json-file | f
 }
 
 @test "Verify /run/.containerenv exist" {
-	run_podman run --rm $IMAGE ls -1 /run/.containerenv
-	is "$output" "/run/.containerenv"
-
-	run_podman run --privileged --rm $IMAGE sh -c '. /run/.containerenv; echo $engine'
-	is "$output" ".*podman.*" "failed to identify engine"
-
-	run_podman run --privileged  --name "testcontainerenv" --rm $IMAGE sh -c '. /run/.containerenv; echo $name'
-	is "$output" ".*testcontainerenv.*"
-
-	run_podman run --privileged  --rm $IMAGE sh -c '. /run/.containerenv; echo $image'
-	is "$output" ".*$IMAGE.*" "failed to idenitfy image"
-
-	run_podman run --privileged --rm $IMAGE sh -c '. /run/.containerenv; echo $rootless'
-	# FIXME: on some CI systems, 'run --privileged' emits a spurious
-	# warning line about dup devices. Ignore it.
-	remove_same_dev_warning
-	if is_rootless; then
-		is "$output" "1"
-	else
-		is "$output" "0"
-	fi
+    # Nonprivileged container: file exists, but must be empty
+    run_podman run --rm $IMAGE stat -c '%s' /run/.containerenv
+    is "$output" "0" "file size of /run/.containerenv, nonprivileged"
+
+    # Prep work: get ID of image; make a cont. name; determine if we're rootless
+    run_podman inspect --format '{{.ID}}' $IMAGE
+    local iid="$output"
+
+    random_cname=c$(random_string 15 | tr A-Z a-z)
+    local rootless=0
+    if is_rootless; then
+        rootless=1
+    fi
+
+    run_podman run --privileged --rm --name $random_cname $IMAGE \
+               sh -c '. /run/.containerenv; echo $engine; echo $name; echo $image; echo $id; echo $imageid; echo $rootless'
+
+    # FIXME: on some CI systems, 'run --privileged' emits a spurious
+    # warning line about dup devices. Ignore it.
+    remove_same_dev_warning
+
+    is "${lines[0]}" "podman-.*"      'containerenv : $engine'
+    is "${lines[1]}" "$random_cname"  'containerenv : $name'
+    is "${lines[2]}" "$IMAGE"         'containerenv : $image'
+    is "${lines[3]}" "[0-9a-f]\{64\}" 'containerenv : $id'
+    is "${lines[4]}" "$iid"           'containerenv : $imageid'
+    is "${lines[5]}" "$rootless"      'containerenv : $rootless'
 }
 
 @test "podman run with --net=host and --port prints warning" {

test/system/040-ps.bats

@@ -82,4 +82,43 @@ load helpers
     run_podman rm -a
 }
 
+@test "podman ps -a --storage" {
+    skip_if_remote "ps --storage does not work over remote"
+
+    # Setup: ensure that we have no hidden storage containers
+    run_podman ps --storage -a
+    is "${#lines[@]}" "1" "setup check: no storage containers at start of test"
+
+    # Force a buildah timeout; this leaves a buildah container behind
+    PODMAN_TIMEOUT=5 run_podman 124 build -t thiswillneverexist - <<EOF
+FROM $IMAGE
+RUN sleep 30
+EOF
+
+    run_podman ps -a
+    is "${#lines[@]}" "1" "podman ps -a does not see buildah container"
+
+    run_podman ps --storage -a
+    is "${#lines[@]}" "2" "podman ps -a --storage sees buildah container"
+    is "${lines[1]}" \
+       "[0-9a-f]\{12\} \+$IMAGE *buildah .* seconds ago .* storage .* ${PODMAN_TEST_IMAGE_NAME}-working-container" \
+       "podman ps --storage"
+
+    cid="${lines[1]:0:12}"
+
+    # 'rm -a' should be a NOP
+    run_podman rm -a
+    run_podman ps --storage -a
+    is "${#lines[@]}" "2" "podman ps -a --storage sees buildah container"
+
+    # This is what deletes the container
+    # FIXME: why doesn't "podman rm --storage $cid" do anything?
+    run_podman rm -f "$cid"
+
+    run_podman ps --storage -a
+    is "${#lines[@]}" "1" "storage container has been removed"
+}
+
+
+
 # vim: filetype=sh

test/system/260-sdnotify.bats

@@ -100,8 +100,17 @@ function _assert_mainpid_is_conmon() {
     run_podman logs sdnotify_conmon_c
     is "$output" "READY" "\$NOTIFY_SOCKET in container"
 
+    # The 'echo's help us debug failed runs
     run cat $_SOCAT_LOG
-    is "${lines[-1]}" "READY=1" "final output from sdnotify"
+    echo "socat log:"
+    echo "$output"
+
+    # ARGH! 'READY=1' should always be the last output line. But sometimes,
+    # for reasons unknown, we get an extra MAINPID=xxx after READY=1 (#8718).
+    # Who knows if this is a systemd bug, or conmon, or what. I don't
+    # even know where to begin asking. So, to eliminate the test flakes,
+    # we look for READY=1 _anywhere_ in the output, not just the last line.
+    is "$output" ".*READY=1.*" "sdnotify sent READY=1"
 
     _assert_mainpid_is_conmon "${lines[0]}"
 

test/system/helpers.bash

@@ -168,8 +168,11 @@ function run_podman() {
 
     if [ "$status" -eq 124 ]; then
         if expr "$output" : ".*timeout: sending" >/dev/null; then
-            echo "*** TIMED OUT ***"
-            false
+            # It's possible for a subtest to _want_ a timeout
+            if [[ "$expected_rc" != "124" ]]; then
+                echo "*** TIMED OUT ***"
+                false
+            fi
         fi
     fi