From a2276cb4a4017cf9f10ee15fe6811c5c427a5dcc Mon Sep 17 00:00:00 2001
From: yoni206 <yoni206@users.noreply.github.com>
Date: Wed, 13 Sep 2023 00:33:49 +0300
Subject: [PATCH] fixing run_regressions.py so that it does not ignore errors
 in scrubbers (#9370)

run_regressions.py was ignoring errors in scrubber commands.
For example, when using grep -o "--[a-zA-Z-]+" in a scrubber, grep returns an error, but the script ignored it.
Two regressions are fixed accordingly.
---
 .../cli/regress0/nl/proj-issue-425.smt2       |  3 ++-
 .../cli/regress0/options/didyoumean.smt2      |  4 +--
 .../cli/regress0/parser/stdout-diag.smt2      |  2 +-
 test/regress/cli/run_regression.py            | 26 +++++++++++++++++--
 4 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/test/regress/cli/regress0/nl/proj-issue-425.smt2 b/test/regress/cli/regress0/nl/proj-issue-425.smt2
index d7e293489c5..49aedb93205 100644
--- a/test/regress/cli/regress0/nl/proj-issue-425.smt2
+++ b/test/regress/cli/regress0/nl/proj-issue-425.smt2
@@ -1,5 +1,6 @@
 ; COMMAND-LINE: --solve-int-as-bv=5524936381719514648
-; ERROR-SCRUBBER: sed -e '.*Error in option parsing.*/d'
+; EXPECT-ERROR: Error in option parsing
+; ERROR-SCRUBBER: grep -o "Error in option parsing"
 ; DISABLE-TESTER: dump
 ; REQUIRES: no-competition
 ; EXIT: 1
diff --git a/test/regress/cli/regress0/options/didyoumean.smt2 b/test/regress/cli/regress0/options/didyoumean.smt2
index 43c4d5903a4..92acafa7de6 100644
--- a/test/regress/cli/regress0/options/didyoumean.smt2
+++ b/test/regress/cli/regress0/options/didyoumean.smt2
@@ -1,6 +1,6 @@
 ; DISABLE-TESTER: dump
 ; REQUIRES: no-competition
 ; COMMAND-LINE: --input-agnuage
-; ERROR-SCRUBBER: grep -o "--[a-zA-Z-]+"
+; ERROR-SCRUBBER: grep -o "[a-zA-Z-]+"
 ; ERROR-EXPECT: --input-language
-; EXIT: 1
\ No newline at end of file
+; EXIT: 1
diff --git a/test/regress/cli/regress0/parser/stdout-diag.smt2 b/test/regress/cli/regress0/parser/stdout-diag.smt2
index ab409561bfc..06d85d95e40 100644
--- a/test/regress/cli/regress0/parser/stdout-diag.smt2
+++ b/test/regress/cli/regress0/parser/stdout-diag.smt2
@@ -1,5 +1,5 @@
 ; Ensure that no `stdout` or `"stdout"` file is created
-; SCRUBBER: ls stdout || ls \"stdout\" || echo success
+; SCRUBBER: [ -f  stdout ] || [ -f \"stdout\" ] || echo success
 ; EXPECT: success
 (set-option :global-declarations true)
 (set-option :diagnostic-output-channel "stdout")
diff --git a/test/regress/cli/run_regression.py b/test/regress/cli/run_regression.py
index 15ecc22e46a..40c8f014c8a 100755
--- a/test/regress/cli/run_regression.py
+++ b/test/regress/cli/run_regression.py
@@ -533,6 +533,17 @@ def get_cvc5_features(cvc5_binary):
 
     return features, disabled_features
 
+def check_scrubber(scrubber_error, scrubber):
+    if len(scrubber_error) != 0:
+        print_error("The scrubber's error output is not empty")
+        print()
+        print("  Command: {}".format(scrubber))
+        print()
+        print("  Error output")
+        print("  " + "=" * 78)
+        print(scrubber_error)
+        print("  " + "=" * 78)
+        return EXIT_FAILURE
 
 def run_benchmark(benchmark_info):
     """Runs cvc5 on a benchmark with the given `benchmark_info`. It runs on the
@@ -556,20 +567,31 @@ def run_benchmark(benchmark_info):
     )
 
     # If a scrubber command has been specified then apply it to the output.
+    scrubber_error = ""
     if benchmark_info.scrubber:
-        output, _, _ = run_process(
+        output, scrubber_error, _ = run_process(
             benchmark_info.scrubber,
             benchmark_info.benchmark_dir,
             benchmark_info.timeout,
             output,
         )
+    # Make sure that the scrubber itself did not print anything to its error output
+    check_result =  check_scrubber(scrubber_error, benchmark_info.scrubber)
+    if check_result != None:
+      return check_result
+    
+    scrubber_error = ""
     if benchmark_info.error_scrubber:
-        error, _, _ = run_process(
+        error, scrubber_error, _ = run_process(
             benchmark_info.error_scrubber,
             benchmark_info.benchmark_dir,
             benchmark_info.timeout,
             error,
         )
+    # Make sure that the error scrubber itself did not print anything to its error output
+    check_result =  check_scrubber(scrubber_error, benchmark_info.error_scrubber)
+    if check_result != None:
+      return check_result
 
     # Popen in Python 3 returns a bytes object instead of a string for
     # stdout/stderr.