From ec1c57cb999086d7aa2db970c0c8e2838ff8b510 Mon Sep 17 00:00:00 2001
From: Timothee Cour <timothee.cour2@gmail.com>
Date: Thu, 6 May 2021 14:53:11 -0700
Subject: [PATCH] revert workarounds from
 https://github.com/nim-lang/Nim/pull/16698 and add allowPrefixMatch optional
 param to greedyOrderedSubsetLines

---
 testament/lib/stdtest/testutils.nim         | 10 +++-
 testament/tests/shouldfail/tccodecheck.nim  |  1 -
 testament/tests/shouldfail/tcolumn.nim      |  1 -
 testament/tests/shouldfail/terrormsg.nim    |  1 -
 testament/tests/shouldfail/texitcode1.nim   |  1 -
 testament/tests/shouldfail/tfile.nim        |  1 -
 testament/tests/shouldfail/tline.nim        |  1 -
 testament/tests/shouldfail/tmaxcodesize.nim |  1 -
 testament/tests/shouldfail/tnimout.nim      |  1 -
 testament/tests/shouldfail/tnimoutfull.nim  |  1 -
 testament/tests/shouldfail/toutput.nim      |  1 -
 testament/tests/shouldfail/toutputsub.nim   |  1 -
 testament/tests/shouldfail/treject.nim      |  1 -
 testament/tests/shouldfail/tsortoutput.nim  |  1 -
 testament/tests/shouldfail/ttimeout.nim     |  1 -
 testament/tests/shouldfail/tvalgrind.nim    |  1 -
 tests/misc/trunner.nim                      |  2 +
 tests/testament/tshould_not_work.nim        | 51 +++++++++++++--------
 18 files changed, 41 insertions(+), 37 deletions(-)

diff --git a/testament/lib/stdtest/testutils.nim b/testament/lib/stdtest/testutils.nim
index 241ab177060f1..6d8620dc7fda1 100644
--- a/testament/lib/stdtest/testutils.nim
+++ b/testament/lib/stdtest/testutils.nim
@@ -29,11 +29,17 @@ template flakyAssert*(cond: untyped, msg = "", notifySuccess = true) =
 when not defined(js):
   import std/strutils
 
-  proc greedyOrderedSubsetLines*(lhs, rhs: string): bool =
+  proc greedyOrderedSubsetLines*(lhs, rhs: string, allowPrefixMatch = false): bool =
     ## Returns true if each stripped line in `lhs` appears in rhs, using a greedy matching.
+    # xxx improve error reporting by showing the last matched pair
     iterator splitLinesClosure(): string {.closure.} =
       for line in splitLines(rhs.strip):
         yield line
+    template isMatch(lhsi, rhsi): bool =
+      if allowPrefixMatch:
+        startsWith(rhsi, lhsi):
+      else:
+        lhsi == rhsi
 
     var rhsIter = splitLinesClosure
     var currentLine = strip(rhsIter())
@@ -41,7 +47,7 @@ when not defined(js):
     for line in lhs.strip.splitLines:
       let line = line.strip
       if line.len != 0:
-        while line != currentLine:
+        while not isMatch(line, currentLine):
           currentLine = strip(rhsIter())
           if rhsIter.finished:
             return false
diff --git a/testament/tests/shouldfail/tccodecheck.nim b/testament/tests/shouldfail/tccodecheck.nim
index 7b5f0cce682dc..477da1e23066c 100644
--- a/testament/tests/shouldfail/tccodecheck.nim
+++ b/testament/tests/shouldfail/tccodecheck.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   ccodecheck: "baz"
 """
 
diff --git a/testament/tests/shouldfail/tcolumn.nim b/testament/tests/shouldfail/tcolumn.nim
index b79ec52a45eb6..809ddec74f826 100644
--- a/testament/tests/shouldfail/tcolumn.nim
+++ b/testament/tests/shouldfail/tcolumn.nim
@@ -1,6 +1,5 @@
 discard """
   errormsg: "undeclared identifier: 'undeclared'"
-  targets: "c"
   line: 9
   column: 7
 """
diff --git a/testament/tests/shouldfail/terrormsg.nim b/testament/tests/shouldfail/terrormsg.nim
index e690352357c7d..6c130d107b5ed 100644
--- a/testament/tests/shouldfail/terrormsg.nim
+++ b/testament/tests/shouldfail/terrormsg.nim
@@ -1,6 +1,5 @@
 discard """
   errormsg: "wrong error message"
-  targets: "c"
   line: 9
   column: 6
 """
diff --git a/testament/tests/shouldfail/texitcode1.nim b/testament/tests/shouldfail/texitcode1.nim
index e5e06157831c7..605f046dba156 100644
--- a/testament/tests/shouldfail/texitcode1.nim
+++ b/testament/tests/shouldfail/texitcode1.nim
@@ -1,4 +1,3 @@
 discard """
-  targets: "c"
   exitcode: 1
 """
diff --git a/testament/tests/shouldfail/tfile.nim b/testament/tests/shouldfail/tfile.nim
index 9463882f9b905..b40a4f44f4aeb 100644
--- a/testament/tests/shouldfail/tfile.nim
+++ b/testament/tests/shouldfail/tfile.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   errormsg: "undeclared identifier: 'undefined'"
   file: "notthisfile.nim"
 """
diff --git a/testament/tests/shouldfail/tline.nim b/testament/tests/shouldfail/tline.nim
index 7f7e90896d2d7..fe782eb03d954 100644
--- a/testament/tests/shouldfail/tline.nim
+++ b/testament/tests/shouldfail/tline.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   errormsg: "undeclared identifier: 'undeclared'"
   line: 10
   column: 6
diff --git a/testament/tests/shouldfail/tmaxcodesize.nim b/testament/tests/shouldfail/tmaxcodesize.nim
index 9e2bd9cfb4481..92022ee975598 100644
--- a/testament/tests/shouldfail/tmaxcodesize.nim
+++ b/testament/tests/shouldfail/tmaxcodesize.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   maxcodesize: 1
 """
 
diff --git a/testament/tests/shouldfail/tnimout.nim b/testament/tests/shouldfail/tnimout.nim
index 832f134b0afdb..0a65bfb701234 100644
--- a/testament/tests/shouldfail/tnimout.nim
+++ b/testament/tests/shouldfail/tnimout.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   nimout: "Hello World!"
   action: compile
 """
diff --git a/testament/tests/shouldfail/tnimoutfull.nim b/testament/tests/shouldfail/tnimoutfull.nim
index 3349ceedf038c..4fc93f6d29bda 100644
--- a/testament/tests/shouldfail/tnimoutfull.nim
+++ b/testament/tests/shouldfail/tnimoutfull.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   nimout: '''
 msg1
 msg2
diff --git a/testament/tests/shouldfail/toutput.nim b/testament/tests/shouldfail/toutput.nim
index 0fa4d72783832..eaf9e865211a3 100644
--- a/testament/tests/shouldfail/toutput.nim
+++ b/testament/tests/shouldfail/toutput.nim
@@ -1,5 +1,4 @@
 discard """
-  targets: "c"
   output: '''
   done
   '''
diff --git a/testament/tests/shouldfail/toutputsub.nim b/testament/tests/shouldfail/toutputsub.nim
index b34f3a8f2c285..47324ecee8cd3 100644
--- a/testament/tests/shouldfail/toutputsub.nim
+++ b/testament/tests/shouldfail/toutputsub.nim
@@ -1,6 +1,5 @@
 discard """
   outputsub: "something else"
-  targets: "c"
 """
 
 echo "Hello World!"
diff --git a/testament/tests/shouldfail/treject.nim b/testament/tests/shouldfail/treject.nim
index 395dc425113e7..1e7258f708b75 100644
--- a/testament/tests/shouldfail/treject.nim
+++ b/testament/tests/shouldfail/treject.nim
@@ -1,6 +1,5 @@
 discard """
   action: "reject"
-  targets: "c"
 """
 
 # Because we set action="reject", we expect this line not to compile. But the
diff --git a/testament/tests/shouldfail/tsortoutput.nim b/testament/tests/shouldfail/tsortoutput.nim
index 0c165d21b3349..7c77d16e0234d 100644
--- a/testament/tests/shouldfail/tsortoutput.nim
+++ b/testament/tests/shouldfail/tsortoutput.nim
@@ -1,6 +1,5 @@
 discard """
   sortoutput: true
-  targets: "c"
   output: '''
   2
   1
diff --git a/testament/tests/shouldfail/ttimeout.nim b/testament/tests/shouldfail/ttimeout.nim
index 8ffd71aaadf3d..fd3e1a598a2f1 100644
--- a/testament/tests/shouldfail/ttimeout.nim
+++ b/testament/tests/shouldfail/ttimeout.nim
@@ -1,6 +1,5 @@
 discard """
   timeout: "0.1"
-  targets: "c"
 """
 
 import os
diff --git a/testament/tests/shouldfail/tvalgrind.nim b/testament/tests/shouldfail/tvalgrind.nim
index 5502705b3ce78..d551ff12ea78f 100644
--- a/testament/tests/shouldfail/tvalgrind.nim
+++ b/testament/tests/shouldfail/tvalgrind.nim
@@ -1,6 +1,5 @@
 discard """
   valgrind: true
-  targets: "c"
   cmd: "nim $target --gc:arc -d:useMalloc $options $file"
 """
 
diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim
index 014373dfb1235..1b679d92c5046 100644
--- a/tests/misc/trunner.nim
+++ b/tests/misc/trunner.nim
@@ -6,6 +6,8 @@ discard """
 ## tests that don't quite fit the mold and are easier to handle via `execCmdEx`
 ## A few others could be added to here to simplify code.
 ## Note: this test is a bit slow but tests a lot of things; please don't disable.
+## Note: if needed, we could use `matrix: "-d:case1; -d:case2"` to split this
+## into several independent tests while retaining the common test helpers.
 
 import std/[strformat,os,osproc,unittest,compilesettings]
 from std/sequtils import toSeq,mapIt
diff --git a/tests/testament/tshould_not_work.nim b/tests/testament/tshould_not_work.nim
index 2777bfe95e3fa..3d6233f2693d2 100644
--- a/tests/testament/tshould_not_work.nim
+++ b/tests/testament/tshould_not_work.nim
@@ -1,39 +1,50 @@
-discard """
-cmd: "testament/testament --directory:testament --colors:off --backendLogging:off --nim:$nim category shouldfail"
-action: compile
-nimout: '''
-FAIL: tests/shouldfail/tccodecheck.nim c
+const expected = """
+FAIL: tests/shouldfail/tccodecheck.nim
 Failure: reCodegenFailure
 Expected:
 baz
-FAIL: tests/shouldfail/tcolumn.nim c
+FAIL: tests/shouldfail/tcolumn.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/terrormsg.nim c
+FAIL: tests/shouldfail/terrormsg.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/texitcode1.nim c
+FAIL: tests/shouldfail/texitcode1.nim
 Failure: reExitcodesDiffer
-FAIL: tests/shouldfail/tfile.nim c
+FAIL: tests/shouldfail/tfile.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tline.nim c
+FAIL: tests/shouldfail/tline.nim
 Failure: reLinesDiffer
-FAIL: tests/shouldfail/tmaxcodesize.nim c
+FAIL: tests/shouldfail/tmaxcodesize.nim
 Failure: reCodegenFailure
 max allowed size: 1
-FAIL: tests/shouldfail/tnimout.nim c
+FAIL: tests/shouldfail/tnimout.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/tnimoutfull.nim c
+FAIL: tests/shouldfail/tnimoutfull.nim
 Failure: reMsgsDiffer
-FAIL: tests/shouldfail/toutput.nim c
+FAIL: tests/shouldfail/toutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/toutputsub.nim c
+FAIL: tests/shouldfail/toutputsub.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/treject.nim c
+FAIL: tests/shouldfail/treject.nim
 Failure: reFilesDiffer
-FAIL: tests/shouldfail/tsortoutput.nim c
+FAIL: tests/shouldfail/tsortoutput.nim
 Failure: reOutputsDiffer
-FAIL: tests/shouldfail/ttimeout.nim c
+FAIL: tests/shouldfail/ttimeout.nim
 Failure: reTimeout
-FAIL: tests/shouldfail/tvalgrind.nim c
+FAIL: tests/shouldfail/tvalgrind.nim
 Failure: reExitcodesDiffer
-'''
 """
+
+import std/[os,strformat,osproc]
+import stdtest/testutils
+
+proc main =
+  const nim = getCurrentCompilerExe()
+  # TODO: bin/testament instead? like other tools (eg bin/nim, bin/nimsuggest etc)
+  let testamentExe = "testament/testament"
+  let cmd = fmt"{testamentExe} --directory:testament --colors:off --backendLogging:off --nim:{nim} category shouldfail"
+  let (outp, status) = execCmdEx(cmd)
+  doAssert status == 1, $status
+
+  let ok = greedyOrderedSubsetLines(expected, outp, allowPrefixMatch = true)
+  doAssert ok, &"\nexpected:\n{expected}\noutp:\n{outp}"
+main()