Add initial version (#21)

.gitignore

@@ -1,2 +1,3 @@
 tests/*/results.json
 tests/*/Mason.lock
+tests/*/target

Dockerfile

@@ -1,4 +1,4 @@
-FROM chapel/chapel:1.32.0
+FROM chapel/chapel:1.33.0
 
 RUN apt update && apt install -y jq
 

bin/run.sh

@@ -45,8 +45,15 @@ popd > /dev/null
 if [ ${exit_code} -eq 0 ]; then
     jq -n '{version: 1, status: "pass"}' > ${results_file}
 else
+    echo "${test_output}" | grep -q 'compilation failed'
+    if [ $? -eq 1 ]; then
+        status="fail"
+    else
+        status="error"
+    fi
+
     sanitized_test_output=$(echo "${test_output}" | sed 's/Skipping registry update since no dependency found in manifest file.//')
-    jq -n --arg output "${sanitized_test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+    jq -n --arg status "${status}" --arg message "${sanitized_test_output}" '{version: 1, status: $status, message: $message}' > ${results_file}
 fi
 
 echo "${slug}: done"

tests/all-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "\n\n======================================================================\nFAIL tests.chpl: testYearNotDivisibleBy4InCommonYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy2NotDivisibleBy4InCommonYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy4NotDivisibleBy100InLeapYear()\n----------------------------------------------------------------------\nAssertionError: assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy4And5IsStillALeapYear()\n----------------------------------------------------------------------\nAssertionError: assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100ButNotBy3IsStillNotALeapYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy400IsLeapYear()\n----------------------------------------------------------------------\nAssertionError: assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy400ButNotBy125IsStillALeapYear()\n----------------------------------------------------------------------\nAssertionError: assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy200NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n----------------------------------------------------------------------\nRan 9 tests\n\nFAILED (failures = 9 )"
+  "message": "\n\n======================================================================\nFAIL tests.chpl: testYearNotDivisibleBy4InCommonYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:6 - assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy2NotDivisibleBy4InCommonYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:10 - assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy4NotDivisibleBy100InLeapYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:14 - assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy4And5IsStillALeapYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:18 - assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:22 - assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100ButNotBy3IsStillNotALeapYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:26 - assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy400IsLeapYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:30 - assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy400ButNotBy125IsStillALeapYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:34 - assertTrue failed. Given expression is False\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy200NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:38 - assertFalse failed. Given expression is True\n\n----------------------------------------------------------------------\nRan 9 tests\n\nFAILED (failures = 9 )"
 }

tests/empty-file/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
-  "status": "fail",
+  "status": "error",
   "message": "compilation failed for tests.chpl\n\n\n======================================================================\nERROR tests.chpl: tests\n----------------------------------------------------------------------\ntests.chpl failed to compile\nTry running 'mason test --show' for more details\n----------------------------------------------------------------------\nRan 1 test\n\nFAILED (errors = 1 )"
 }

tests/partial-fail/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100ButNotBy3IsStillNotALeapYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy200NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: assertFalse failed. Given expression is True\n\n----------------------------------------------------------------------\nRan 9 tests\n\nFAILED (passed = 6 failures = 3 )"
+  "message": "\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:22 - assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy100ButNotBy3IsStillNotALeapYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:26 - assertFalse failed. Given expression is True\n\n======================================================================\nFAIL tests.chpl: testYearDivisibleBy200NotDivisibleBy400InCommonYear()\n----------------------------------------------------------------------\nAssertionError: in test/tests.chpl:38 - assertFalse failed. Given expression is True\n\n----------------------------------------------------------------------\nRan 9 tests\n\nFAILED (passed = 6 failures = 3 )"
 }

tests/syntax-error/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
-  "status": "fail",
+  "status": "error",
   "message": "compilation failed for tests.chpl\n\n\n======================================================================\nERROR tests.chpl: tests\n----------------------------------------------------------------------\ntests.chpl failed to compile\nTry running 'mason test --show' for more details\n----------------------------------------------------------------------\nRan 1 test\n\nFAILED (errors = 1 )"
 }