fixing run_regressions.py so that it does not ignore errors in scrubb…

…ers (cvc5#9370)

run_regressions.py was ignoring errors in scrubber commands.
For example, when using grep -o "--[a-zA-Z-]+" in a scrubber, grep returns an error, but the script ignored it.
Two regressions are fixed accordingly.

test/regress/cli/regress0/nl/proj-issue-425.smt2

@@ -1,5 +1,6 @@
 ; COMMAND-LINE: --solve-int-as-bv=5524936381719514648
-; ERROR-SCRUBBER: sed -e '.*Error in option parsing.*/d'
+; EXPECT-ERROR: Error in option parsing
+; ERROR-SCRUBBER: grep -o "Error in option parsing"
 ; DISABLE-TESTER: dump
 ; REQUIRES: no-competition
 ; EXIT: 1

test/regress/cli/regress0/options/didyoumean.smt2

@@ -1,6 +1,6 @@
 ; DISABLE-TESTER: dump
 ; REQUIRES: no-competition
 ; COMMAND-LINE: --input-agnuage
-; ERROR-SCRUBBER: grep -o "--[a-zA-Z-]+"
+; ERROR-SCRUBBER: grep -o "[a-zA-Z-]+"
 ; ERROR-EXPECT: --input-language
-; EXIT: 1
+; EXIT: 1

test/regress/cli/regress0/parser/stdout-diag.smt2

@@ -1,5 +1,5 @@
 ; Ensure that no `stdout` or `"stdout"` file is created
-; SCRUBBER: ls stdout || ls \"stdout\" || echo success
+; SCRUBBER: [ -f  stdout ] || [ -f \"stdout\" ] || echo success
 ; EXPECT: success
 (set-option :global-declarations true)
 (set-option :diagnostic-output-channel "stdout")

test/regress/cli/run_regression.py

@@ -533,6 +533,17 @@ def get_cvc5_features(cvc5_binary):
 
     return features, disabled_features
 
+def check_scrubber(scrubber_error, scrubber):
+    if len(scrubber_error) != 0:
+        print_error("The scrubber's error output is not empty")
+        print()
+        print("  Command: {}".format(scrubber))
+        print()
+        print("  Error output")
+        print("  " + "=" * 78)
+        print(scrubber_error)
+        print("  " + "=" * 78)
+        return EXIT_FAILURE
 
 def run_benchmark(benchmark_info):
     """Runs cvc5 on a benchmark with the given `benchmark_info`. It runs on the
@@ -556,20 +567,31 @@ def run_benchmark(benchmark_info):
     )
 
     # If a scrubber command has been specified then apply it to the output.
+    scrubber_error = ""
     if benchmark_info.scrubber:
-        output, _, _ = run_process(
+        output, scrubber_error, _ = run_process(
             benchmark_info.scrubber,
             benchmark_info.benchmark_dir,
             benchmark_info.timeout,
             output,
         )
+    # Make sure that the scrubber itself did not print anything to its error output
+    check_result =  check_scrubber(scrubber_error, benchmark_info.scrubber)
+    if check_result != None:
+      return check_result
+
+    scrubber_error = ""
     if benchmark_info.error_scrubber:
-        error, _, _ = run_process(
+        error, scrubber_error, _ = run_process(
             benchmark_info.error_scrubber,
             benchmark_info.benchmark_dir,
             benchmark_info.timeout,
             error,
         )
+    # Make sure that the error scrubber itself did not print anything to its error output
+    check_result =  check_scrubber(scrubber_error, benchmark_info.error_scrubber)
+    if check_result != None:
+      return check_result
 
     # Popen in Python 3 returns a bytes object instead of a string for
     # stdout/stderr.