System tests: enhance volume test, add debug prints

Volume test: add a sequence of stat()s to confirm that volumes
are mounted as a different device than root.

Network test: add debugging code for containers#11825 (dnsmasq inotify
failure in bodhi only).

Signed-off-by: Ed Santiago <[email protected]>

test/system/070-build.bats

@@ -39,6 +39,7 @@ EOF
     cat >$dockerfile <<EOF
 FROM $IMAGE
 RUN echo $rand_content > /$rand_filename
+VOLUME /a/b/c
 VOLUME ['/etc/foo', '/etc/bar']
 EOF
 
@@ -56,6 +57,25 @@ EOF
 /\[/etc/foo,
 /etc/bar]" "weird VOLUME gets converted to directories with brackets and comma"
 
+    # Now confirm that each volume got a unique device ID
+    run_podman run --rm build_test stat -c '%D' / /a /a/b /a/b/c /\[ /\[/etc /\[/etc/foo, /etc /etc/bar\]
+    # First, the non-volumes should all be the same...
+    is "${lines[0]}" "${lines[1]}" "devnum( / ) = devnum( /a )"
+    is "${lines[0]}" "${lines[2]}" "devnum( / ) = devnum( /a/b )"
+    is "${lines[0]}" "${lines[4]}" "devnum( / ) = devnum( /[ )"
+    is "${lines[0]}" "${lines[5]}" "devnum( / ) = devnum( /[etc )"
+    is "${lines[0]}" "${lines[7]}" "devnum( / ) = devnum( /etc )"
+    is "${lines[6]}" "${lines[8]}" "devnum( /[etc/foo, ) = devnum( /etc/bar] )"
+    # ...then, each volume should be different
+    if [[ "${lines[0]}" = "${lines[3]}" ]]; then
+        die "devnum( / ) (${lines[0]}) = devnum( volume0 ) (${lines[3]}) -- they should differ"
+    fi
+    if [[ "${lines[0]}" = "${lines[6]}" ]]; then
+        die "devnum( / ) (${lines[0]}) = devnum( volume1 ) (${lines[6]}) -- they should differ"
+    fi
+    # FIXME: is this expected? I thought /a/b/c and /[etc/foo, would differ
+    is "${lines[3]}" "${lines[6]}" "devnum( volume0 ) = devnum( volume1 )"
+
     run_podman rmi -f build_test
 }
 

test/system/500-networking.bats

@@ -444,6 +444,14 @@ load helpers
         die "MAC address did not change after podman network disconnect/connect"
     fi
 
+    # FIXME FIXME FIXME: #11825: bodhi tests are failing, remote+rootless only,
+    # with "dnsmasq: failed to create inotify". This error has never occurred
+    # in CI, and Ed has been unable to reproduce it on 1minutetip. This next
+    # line is a suggestion from Paul Holzinger for trying to shed light on
+    # the system context before the failure. This output will be invisible
+    # if the test passes.
+    for foo in /proc/\*/fd/*; do readlink -f $foo; done |grep '^/proc/.*inotify' |cut -d/ -f3 | xargs -I '{}' -- ps --no-headers -o '%p %U %a' -p '{}' |uniq -c |sort -n
+
     # connect a second network
     run_podman network connect $netname2 $cid