TEMPORARY(?) instrumentation for unlinkat-ebusy

Instrument system tests in hopes of tracking down containers#17216, the unlinkat-ebusy-hosed flake. Oh, also, timestamp.awk: timestamps have always been UTC, but add a 'Z' to make it unambiguous. Signed-off-by: Ed Santiago <[email protected]>
edsantiago · May 22, 2023 · 94c65a6 · 94c65a6
1 parent b155106
commit 94c65a6
Show file tree

Hide file tree

Showing 2 changed files with 25 additions and 3 deletions.
diff --git a/contrib/cirrus/timestamp.awk b/contrib/cirrus/timestamp.awk
@@ -7,7 +7,7 @@
 BEGIN {
     STARTTIME=systime()
     printf "[%s] START", strftime("%T")
-    printf " - All [+xxxx] lines that follow are relative to %s.\n", strftime("%FT%T")
+    printf " - All [+xxxx] lines that follow are relative to %s.\n", strftime("%FT%TZ", systime(), 1)
 }
 
 {
@@ -16,5 +16,5 @@ BEGIN {
 
 END {
     printf "[%s] END", strftime("%T")
-    printf " - [%+05ds] total duration since %s\n", systime()-STARTTIME, strftime("%FT%T")
+    printf " - [%+05ds] total duration since %s\n", systime()-STARTTIME, strftime("%FT%TZ", systime(), 1)
 }
diff --git a/test/system/helpers.bash b/test/system/helpers.bash
@@ -45,6 +45,11 @@ fi
 
 # Setup helper: establish a test environment with exactly the images needed
 function basic_setup() {
+    # FIXME FIXME FIXME: remove if #17216 is fixed. See below also.
+    if [[ -e "${BATS_SUITE_TMPDIR}/forget-it" ]]; then
+        skip "everything is hosed, no point in going on"
+    fi
+
     # Clean up all containers
     run_podman rm -t 0 --all --force --ignore
 
@@ -58,7 +63,24 @@ function basic_setup() {
 
     # Clean up all images except those desired
     found_needed_image=
-    run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}'
+    run_podman '?' images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}'
+    # FIXME FIXME FIXME: temporary hack for #17216. If we see the unlinkat-busy
+    # flake, nothing will ever work again.
+    if [[ $status -ne 0 ]]; then
+        if [[ "$output" =~ unlinkat.*busy ]]; then
+            # Signal (see above) to skip all subsequent tests.
+            touch "${BATS_SUITE_TMPDIR}/forget-it"
+            # Gather some debugging info, then fail
+            echo "$_LOG_PROMPT ps auxww --forest"
+            ps auxww --forest
+            echo "$_LOG_PROMPT mount"
+            mount
+            echo "$_LOG_PROMPT lsof /var/lib/containers"
+            lsof /var/lib/containers
+            false
+        fi
+    fi
+
     for line in "${lines[@]}"; do
         set $line
         if [[ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]]; then