Skip to content

Commit

Permalink
Merge pull request containers#17538 from edsantiago/system_test_assert
Browse files Browse the repository at this point in the history 
System tests: assert(): friendlier failure messages
  • Loading branch information
@openshift-merge-robot
openshift-merge-robot authored Feb 17, 2023
2 parents f9af496 + c81fbd5 commit b55df7f
Show file tree
Hide file tree
Showing 2 changed files with 180 additions and 5 deletions.
28 changes: 23 additions & 5 deletions test/system/helpers.bash
View file
Original file line number Diff line number Diff line change
Expand Up @@ -629,15 +629,33 @@ function assert() {

# This is a multi-line message, which may in turn contain multi-line
# output, so let's format it ourself to make it more readable.
local expect_split
mapfile -t expect_split <<<"$expect_string"
local actual_split
IFS=$'\n' read -rd '' -a actual_split <<<"$actual_string" || true
mapfile -t actual_split <<<"$actual_string"

# bash %q is really nice, except for the way it backslashes spaces
local -a expect_split_q
for line in "${expect_split[@]}"; do
local q=$(printf "%q" "$line" | sed -e 's/\\ / /g')
expect_split_q+=("$q")
done
local -a actual_split_q
for line in "${actual_split[@]}"; do
local q=$(printf "%q" "$line" | sed -e 's/\\ / /g')
actual_split_q+=("$q")
done

printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
printf "#| FAIL: %s\n" "$testname" >&2
printf "#| expected: %s'%s'\n" "$op" "$expect_string" >&2
printf "#| actual: %s'%s'\n" "$ws" "${actual_split[0]}" >&2
printf "#| expected: %s%s\n" "$op" "${expect_split_q[0]}" >&2
local line
for line in "${actual_split[@]:1}"; do
printf "#| > %s'%s'\n" "$ws" "$line" >&2
for line in "${expect_split_q[@]:1}"; do
printf "#| > %s%s\n" "$ws" "$line" >&2
done
printf "#| actual: %s%s\n" "$ws" "${actual_split_q[0]}" >&2
for line in "${actual_split_q[@]:1}"; do
printf "#| > %s%s\n" "$ws" "$line" >&2
done
printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2
false
Expand Down
157 changes: 157 additions & 0 deletions test/system/helpers.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -242,5 +242,162 @@ done < <(parse_table "$table")

# END ipv6_to_procfs
###############################################################################
# BEGIN check_assert
#
# This is way, way more complicated than it should be. The purpose is
# to generate readable error messages should any of the tests ever fail.
#

# Args: the last one is "" (expect to pass) or non-"" (expect that as msg).
# All other args are what we feed to assert()
function check_assert() {
local argv=("$@")
testnum=$(expr $testnum + 1)

# Final arg: "" to expect pass, anything else is expected error message
local expect="${argv[-1]}"
unset 'argv[-1]'

# Descriptive test name. If multiline, use sed to make the rest '[...]'
local testname="assert ${argv[*]}"
testname="$(sed -z -e 's/[\r\n].\+/ [...]/' <<<"$testname")"

# HERE WE GO. This is the actual test.
actual=$(assert "${argv[@]}" 2>&1)
status=$?

# Now compare actual to expect.
if [[ -z "$expect" ]]; then
# expect: pass
if [[ $status -eq 0 ]]; then
# got: pass
echo "ok $testnum $testname"
else
# got: fail
echo "not ok $testnum $testname"
echo "# expected success; got:"
local -a actual_split
IFS=$'\n' read -rd '' -a actual_split <<<"$actual" || true
if [[ "${actual_split[0]}" =~ 'vvvvv' ]]; then
unset 'actual_split[0]'
unset 'actual_split[1]'
unset 'actual_split[-1]'
actual_split=("${actual_split[@]}")
fi
for line in "${actual_split[@]}"; do
echo "# $line"
done
rc=1
fi
else
# expect: fail
if [[ $status -eq 0 ]]; then
# got: pass
echo "not ok $testnum $testname"
echo "# expected it to fail, but it passed"
rc=1
else
# Expected failure, got failure. But is it the desired failure?

# Split what we got into lines, and remove the top/bottom borders
local -a actual_split
IFS=$'\n' read -rd '' -a actual_split <<<"$actual" || true
if [[ "${actual_split[0]}" =~ 'vvvvv' ]]; then
unset 'actual_split[0]'
unset 'actual_split[1]'
unset 'actual_split[-1]'
actual_split=("${actual_split[@]}")
fi

# Split the expect string into lines, and remove first if empty
local -a expect_split
IFS=$'\n' read -rd '' -a expect_split <<<"$expect" || true
if [[ -z "${expect_split[0]}" ]]; then
unset 'expect_split[0]'
expect_split=("${expect_split[@]}")
fi

if [[ "${actual_split[*]}" = "${expect_split[*]}" ]]; then
# Yay.
echo "ok $testnum $testname"
else
# Nope. Mismatch between actual and expected output
echo "not ok $testnum $testname"
rc=1

# Ugh, this is complicated. Try to produce a useful err msg.
local n_e=${#expect_split[*]}
local n_a=${#actual_split[*]}
local n_max=${n_e}
if [[ $n_max -lt $n_a ]]; then
n_max=${n_a}
fi
printf "# %-35s | actual\n" "expect"
printf "# ----------------------------------- | ------\n"
for i in $(seq 0 $((${n_max}-1))); do
local e="${expect_split[$i]}"
local a="${actual_split[$i]}"
local same=' '
local eq='='
if [[ "$e" != "$a" ]]; then
same='!'
eq='|'
fi
printf "# %s %-35s %s %s\n" "$same" "$e" "$eq" "$a"
done
fi
fi
fi
}

# Positive tests
check_assert "a" = "a" ""
check_assert "abc" =~ "a" ""
check_assert "abc" =~ "b" ""
check_assert "abc" =~ "c" ""
check_assert "abc" =~ "a.*c" ""
check_assert "a" != "b" ""

# Simple Failure tests
check_assert "a" = "b" "
#| expected: = b
#| actual: a"

# This is the one that triggered #17509
expect="abcd efg
hijk lmnop"
actual="abcd efg
hijk lmnop"
check_assert "$actual" = "$expect" "
#| expected: = abcd efg
#| > hijk lmnop
#| actual: abcd efg
#| > ''
#| > hijk lmnop"

# Undesired carriage returns
cr=$'\r'
expect="this is line 1
this is line 2"
actual="this is line 1$cr
this is line 2$cr"
check_assert "$actual" = "$expect" "
#| expected: = this is line 1
#| > this is line 2
#| actual: \$'this is line 1\r'
#| > \$'this is line 2\r'"

# Anchored expressions; the 2nd and 3rd are 15 and 17 characters, not 16
check_assert "0123456789abcdef" =~ "^[0-9a-f]{16}\$" ""
check_assert "0123456789abcde" =~ "^[0-9a-f]{16}\$" "
#| expected: =~ \^\[0-9a-f\]\{16\}\\$
#| actual: 0123456789abcde"
check_assert "0123456789abcdeff" =~ "^[0-9a-f]{16}\$" "
#| expected: =~ \^\[0-9a-f\]\{16\}\\$
#| actual: 0123456789abcdeff"

# END check_assert
###############################################################################

exit $rc

0 comments on commit b55df7f

Please sign in to comment.