Merge pull request #8128 from yanrayw/2.28-7094-collect-compatsh-test…

…-cases

Backport 2.28: check_test_cases.py: support to collect test cases for compat.sh

tests/compat.sh

@@ -128,10 +128,40 @@ print_usage() {
     printf "            \tAlso available: GnuTLS (needs v3.2.15 or higher)\n"
     printf "  -M|--memcheck\tCheck memory leaks and errors.\n"
     printf "  -v|--verbose\tSet verbose output.\n"
+    printf "     --list-test-case\tList all potential test cases (No Execution)\n"
     printf "     --outcome-file\tFile where test outcomes are written\n"
     printf "                   \t(default: \$MBEDTLS_TEST_OUTCOME_FILE, none if empty)\n"
 }
 
+# print_test_case <CLIENT> <SERVER> <STANDARD_CIPHER_SUITE>
+print_test_case() {
+    for i in $3; do
+        uniform_title $1 $2 $i
+        echo $TITLE
+    done
+}
+
+# list_test_case lists all potential test cases in compat.sh without execution
+list_test_case() {
+    for MODE in $MODES; do
+        for TYPE in $TYPES; do
+            for VERIFY in $VERIFIES; do
+                VERIF=$(echo $VERIFY | tr '[:upper:]' '[:lower:]')
+                reset_ciphersuites
+                add_common_ciphersuites
+                add_openssl_ciphersuites
+                add_gnutls_ciphersuites
+                add_mbedtls_ciphersuites
+                print_test_case m O "$O_CIPHERS"
+                print_test_case O m "$O_CIPHERS"
+                print_test_case m G "$G_CIPHERS"
+                print_test_case G m "$G_CIPHERS"
+                print_test_case m m "$M_CIPHERS"
+            done
+        done
+    done
+}
+
 get_options() {
     while [ $# -gt 0 ]; do
         case "$1" in
@@ -159,6 +189,12 @@ get_options() {
             -M|--memcheck)
                 MEMCHECK=1
                 ;;
+            # Please check scripts/check_test_cases.py correspondingly
+            # if you have to modify option, --list-test-case
+            --list-test-case)
+                list_test_case
+                exit $?
+                ;;
             --outcome-file)
                 shift; MBEDTLS_TEST_OUTCOME_FILE=$1
                 ;;
@@ -1199,15 +1235,21 @@ report_fail() {
     fi
 }
 
+# uniform_title <CLIENT> <SERVER> <STANDARD_CIPHER_SUITE>
+# $TITLE is considered as test case description for both --list-test-case and
+# MBEDTLS_TEST_OUTCOME_FILE. This function aims to control the format of
+# each test case description.
+uniform_title() {
+    TITLE="$1->$2 $MODE,$VERIF $3"
+}
+
 # run_client <name> <cipher>
 run_client() {
     # announce what we're going to do
     TESTS=$(( $TESTS + 1 ))
-    TITLE="`echo $1 | head -c1`->`echo $SERVER_NAME | head -c1`"
-    TITLE="$TITLE $MODE,$VERIF $2"
-    printf "%s " "$TITLE"
-    LEN=$(( 72 - `echo "$TITLE" | wc -c` ))
-    for i in `seq 1 $LEN`; do printf '.'; done; printf ' '
+    uniform_title "${1%"${1#?}"}" "${SERVER_NAME%"${SERVER_NAME#?}"}" $2
+    DOTS72="........................................................................"
+    printf "%s %.*s " "$TITLE" "$((71 - ${#TITLE}))" "$DOTS72"
 
     # should we skip?
     if [ "X$SKIP_NEXT" = "XYES" ]; then

tests/scripts/check_test_cases.py

@@ -25,6 +25,7 @@
 import glob
 import os
 import re
+import subprocess
 import sys
 
 class Results:
@@ -111,6 +112,19 @@ def walk_ssl_opt_sh(self, file_name):
                 self.process_test_case(descriptions,
                                        file_name, line_number, description)
 
+    def walk_compat_sh(self, file_name):
+        """Iterate over the test cases compat.sh with a similar format."""
+        descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none
+        compat_cmd = ['sh', file_name, '--list-test-case']
+        compat_output = subprocess.check_output(compat_cmd)
+        # Assume compat.sh is responsible for printing identical format of
+        # test case description between --list-test-case and its OUTCOME.CSV
+        description = compat_output.strip().split(b'\n')
+        # idx indicates the number of test case since there is no line number
+        # in `compat.sh` for each test case.
+        for idx, descrip in enumerate(description):
+            self.process_test_case(descriptions, file_name, idx, descrip)
+
     @staticmethod
     def collect_test_directories():
         """Get the relative path for the TLS and Crypto test directories."""
@@ -133,6 +147,9 @@ def walk_all(self):
             ssl_opt_sh = os.path.join(directory, 'ssl-opt.sh')
             if os.path.exists(ssl_opt_sh):
                 self.walk_ssl_opt_sh(ssl_opt_sh)
+            compat_sh = os.path.join(directory, 'compat.sh')
+            if os.path.exists(compat_sh):
+                self.walk_compat_sh(compat_sh)
 
 class TestDescriptions(TestDescriptionExplorer):
     """Collect the available test cases."""