diff --git a/docs/architecture/testing/test-framework.md b/docs/architecture/testing/test-framework.md index c53795174444..8baa3901f0a9 100644 --- a/docs/architecture/testing/test-framework.md +++ b/docs/architecture/testing/test-framework.md @@ -30,17 +30,23 @@ Each test case has a description which succinctly describes for a human audience #### SSL test case descriptions -Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_tests`. +Each test case in `ssl-opt.sh` has a description which succinctly describes for a human audience what the test does. The test description is the first parameter to `run_test`. The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, the description must be written on the same line as `run_test`, in double quotes, for the sake of `check_test_cases.py`. +### SSL cipher suite tests + +Each test case in `compat.sh` has a description which succinctly describes for a human audience what the test does. The test description is `$TITLE` defined in `run_client`. + +The same rules and guidelines apply as for [unit test descriptions](#unit-test-descriptions). In addition, failure cause in `compat.sh` is not classified as `ssl-opt.sh`, so the information of failed log files are followed as prompt. + ## Running tests ### Outcome file #### Generating an outcome file -Unit tests and `ssl-opt.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file. +Unit tests, `ssl-opt.sh` and `compat.sh` record the outcome of each test case in a **test outcome file**. This feature is enabled if the environment variable `MBEDTLS_TEST_OUTCOME_FILE` is set. Set it to the path of the desired file. If you run `all.sh --outcome-file test-outcome.csv`, this collects the outcome of all the test cases in `test-outcome.csv`. @@ -52,7 +58,7 @@ The outcome file has 6 fields: * **Platform**: a description of the platform, e.g. `Linux-x86_64` or `Linux-x86_64-gcc7-msan`. * **Configuration**: a unique description of the configuration (`config.h`). -* **Test suite**: `test_suite_xxx` or `ssl-opt`. +* **Test suite**: `test_suite_xxx`, `ssl-opt` or `compat`. * **Test case**: the description of the test case. * **Result**: one of `PASS`, `SKIP` or `FAIL`. * **Cause**: more information explaining the result. diff --git a/tests/compat.sh b/tests/compat.sh index e7f9d4981cb0..75d5461599a5 100755 --- a/tests/compat.sh +++ b/tests/compat.sh @@ -30,6 +30,11 @@ set -u # where it may output seemingly unlimited length error logs. ulimit -f 20971520 +ORIGINAL_PWD=$PWD +if ! cd "$(dirname "$0")"; then + exit 125 +fi + # initialise counters TESTS=0 FAILED=0 @@ -77,6 +82,17 @@ else PEER_GNUTLS="" fi +guess_config_name() { + if git diff --quiet ../include/mbedtls/config.h 2>/dev/null; then + echo "default" + else + echo "unknown" + fi +} +: ${MBEDTLS_TEST_OUTCOME_FILE=} +: ${MBEDTLS_TEST_CONFIGURATION:="$(guess_config_name)"} +: ${MBEDTLS_TEST_PLATFORM:="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} + # default values for options # /!\ keep this synchronised with: # - basic-build-test.sh @@ -112,6 +128,8 @@ print_usage() { printf " \tAlso available: GnuTLS (needs v3.2.15 or higher)\n" printf " -M|--memcheck\tCheck memory leaks and errors.\n" printf " -v|--verbose\tSet verbose output.\n" + printf " --outcome-file\tFile where test outcomes are written\n" + printf " \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n" } get_options() { @@ -141,6 +159,9 @@ get_options() { -M|--memcheck) MEMCHECK=1 ;; + --outcome-file) + shift; MBEDTLS_TEST_OUTCOME_FILE=$1 + ;; -h|--help) print_usage exit 0 @@ -1145,6 +1166,39 @@ wait_client_done() { echo "EXIT: $EXIT" >> $CLI_OUT } +# record_outcome [] +record_outcome() { + echo "$1" + if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ]; then + # The test outcome file has the format (in single line): + # platform;configuration; + # test suite name;test case description; + # PASS/FAIL/SKIP;[failure cause] + printf '%s;%s;%s;%s;%s;%s\n' \ + "$MBEDTLS_TEST_PLATFORM" "$MBEDTLS_TEST_CONFIGURATION" \ + "compat" "$TITLE" \ + "$1" "${2-}" \ + >> "$MBEDTLS_TEST_OUTCOME_FILE" + fi +} + +# display additional information if test case fails +report_fail() { + FAIL_PROMPT="outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log" + record_outcome "FAIL" "$FAIL_PROMPT" + cp $SRV_OUT c-srv-${TESTS}.log + cp $CLI_OUT c-cli-${TESTS}.log + echo " ! $FAIL_PROMPT" + + if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then + echo " ! server output:" + cat c-srv-${TESTS}.log + echo " ! ===================================================" + echo " ! client output:" + cat c-cli-${TESTS}.log + fi +} + # run_client run_client() { # announce what we're going to do @@ -1158,7 +1212,7 @@ run_client() { # should we skip? if [ "X$SKIP_NEXT" = "XYES" ]; then SKIP_NEXT="NO" - echo "SKIP" + record_outcome "SKIP" SKIPPED=$(( $SKIPPED + 1 )) return fi @@ -1252,26 +1306,14 @@ run_client() { # report and count result case $RESULT in "0") - echo PASS + record_outcome "PASS" ;; "1") - echo SKIP + record_outcome "SKIP" SKIPPED=$(( $SKIPPED + 1 )) ;; "2") - echo FAIL - cp $SRV_OUT c-srv-${TESTS}.log - cp $CLI_OUT c-cli-${TESTS}.log - echo " ! outputs saved to c-srv-${TESTS}.log, c-cli-${TESTS}.log" - - if [ "${LOG_FAILURE_ON_STDOUT:-0}" != 0 ]; then - echo " ! server output:" - cat c-srv-${TESTS}.log - echo " ! ===================================================" - echo " ! client output:" - cat c-cli-${TESTS}.log - fi - + report_fail FAILED=$(( $FAILED + 1 )) ;; esac @@ -1283,13 +1325,16 @@ run_client() { # MAIN # -if cd $( dirname $0 ); then :; else - echo "cd $( dirname $0 ) failed" >&2 - exit 1 -fi - get_options "$@" +# Make the outcome file path relative to the original directory, not +# to .../tests +case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) + MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE" + ;; +esac + # sanity checks, avoid an avalanche of errors if [ ! -x "$M_SRV" ]; then echo "Command '$M_SRV' is not an executable file" >&2 diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 5a563ab58b61..3a27aac5c45f 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -167,6 +167,9 @@ get_options() { -p|--preserve-logs) PRESERVE_LOGS=1 ;; + --outcome-file) + shift; MBEDTLS_TEST_OUTCOME_FILE=$1 + ;; --port) shift; SRV_PORT=$1 ;; @@ -190,14 +193,6 @@ get_options() { done } -# Make the outcome file path relative to the original directory, not -# to .../tests -case "$MBEDTLS_TEST_OUTCOME_FILE" in - [!/]*) - MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE" - ;; -esac - # Read boolean configuration options from config.h for easy and quick # testing. Skip non-boolean options (with something other than spaces # and a comment after "#define SYMBOL"). The variable contains a @@ -1392,6 +1387,14 @@ cleanup() { get_options "$@" +# Make the outcome file path relative to the original directory, not +# to .../tests +case "$MBEDTLS_TEST_OUTCOME_FILE" in + [!/]*) + MBEDTLS_TEST_OUTCOME_FILE="$ORIGINAL_PWD/$MBEDTLS_TEST_OUTCOME_FILE" + ;; +esac + # Optimize filters: if $FILTER and $EXCLUDE can be expressed as shell # patterns rather than regular expressions, use a case statement instead # of calling grep. To keep the optimizer simple, it is incomplete and only