Add basic error case test support

Makefile

@@ -33,7 +33,8 @@ lib-cov:
 
 test: test-unit
 
-test-all: test-bdd test-tdd test-qunit test-exports test-unit test-grep test-jsapi test-compilers test-glob
+test-all: test-bdd test-tdd test-qunit test-exports test-unit test-grep test-jsapi test-compilers \
+  test-glob test-error
 
 test-jsapi:
 	@node test/jsapi
@@ -104,6 +105,9 @@ test-async-only:
 test-glob:
 	@./test/acceptance/glob/glob.sh
 
+test-error:
+	@./test/acceptance/error/error.sh
+
 non-tty:
 	@./bin/mocha \
 		--reporter dot \

test/acceptance/error/context.js

@@ -0,0 +1,15 @@
+
+// Known issue: #270
+// This will currently fail and stop further execution of mocha entirely.
+// describe('failure during beforeEach', function(){
+//    beforeEach(function(){
+//      throw new Error("Failure during beforeEach");
+//    })
+//
+//   it('should fail during beforeEach #1', function(){
+//   })
+//
+//   it('should fail during beforeEach #2', function(){
+//   })
+// })
+

test/acceptance/error/error.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+REL_SCRIPT_DIR="`dirname \"$0\"`"
+SCRIPT_DIR="`( cd \"$REL_SCRIPT_DIR\" && pwd )`"
+CALL_MOCHA=$SCRIPT_DIR/../../../bin/mocha
+ERRORCASE_ROOT=$SCRIPT_DIR
+
+ERRORCASE_FILES=./*.js
+ERRORCASE_OUTPUT=/tmp/mocha-errorcase.txt
+ERRORCASE_RESULT_PATTERNFILE=$SCRIPT_DIR/match-pattern.txt
+ERRORCASE_EXCLUDE_PATTERNFILE=$SCRIPT_DIR/exclude-pattern.txt
+
+cd $ERRORCASE_ROOT || {
+    echo Could not cd to $SCRIPT_DIR from `pwd`
+    exit 1
+}
+
+$CALL_MOCHA -R json-stream $ERRORCASE_FILES > $ERRORCASE_OUTPUT && {
+    echo  "$ERRORCASE_FILES" in `pwd` should return failing exit code.
+    exit 1
+}
+
+
+# All lines must match one of the following patterns.
+# start line
+# Cases which should fail should include "should fail" in their description.
+# Cases which should pass should include "should pass" in their description.
+# end line with count
+# NOTE: Update the numbers in end line as needed to match expected results.
+#     This is a bit crude but sufficient for now.
+#     This also makes the pattern file a need to change with every added test.  Bummer.
+
+# This process is a little brittle - regex being what it is.
+# Sanity check the pattern file against output that should never report a match.
+echo '["end",{"suites":0,"tests":0,"passes":0,"pending":0,"failures":0,"start":"1999-12-31T23:59:58.000Z","end":"1999-12-31T23:59:59.000Z","duration":0}]' | egrep -f $ERRORCASE_RESULT_PATTERNFILE  && {
+    echo
+    echo ======================================================================
+    echo Sanity check on $ERRORCASE_RESULT_PATTERNFILE reported match where none was expected.
+    echo Either your environment is incorrectly configured or the last edit to the pattern file was bad.
+    echo ======================================================================
+    exit 1
+}
+
+# check for excluded patterns
+cat $ERRORCASE_OUTPUT | egrep -f $ERRORCASE_EXCLUDE_PATTERNFILE && {
+    echo
+    echo ======================================================================
+    echo Results for "$ERRORCASE_FILES" in `pwd` do not match expected.
+    echo ======================================================================
+    echo
+    echo ======================================================================
+    echo The above lines match one of the excluded patterns in "$ERRORCASE_EXCLUDE_PATTERNFILE":
+    echo
+    cat $ERRORCASE_EXCLUDE_PATTERNFILE
+    echo
+    echo
+    echo One of the following has happened:
+    echo    Test marked "should not run" reported a result
+    echo
+    echo Full json output may be found at "$ERRORCASE_OUTPUT".
+    echo
+    echo To see general test output run the following from `pwd`:
+    echo $CALL_MOCHA "$ERRORCASE_FILES"
+    echo ======================================================================
+    exit 1
+}
+
+# check for required patterns
+cat $ERRORCASE_OUTPUT | egrep -v -f $ERRORCASE_RESULT_PATTERNFILE && {
+    echo
+    echo ======================================================================
+    echo Results for "$ERRORCASE_FILES" in `pwd` do not match expected.
+    echo ======================================================================
+    echo
+    echo ======================================================================
+    echo The above lines do not match any of the patterns in "$ERRORCASE_RESULT_PATTERNFILE":
+    echo
+    cat $ERRORCASE_RESULT_PATTERNFILE
+    echo
+    echo
+    echo One of the following has happened:
+    echo    Run did not start
+    echo    Test marked "should pass" reported fail
+    echo    Test marked "should fail" reported pass
+    echo    Tests were added, changed, or removed - the counts in the "end" line need to change
+    echo
+    echo Full output may be found at "$ERRORCASE_OUTPUT".
+    echo
+    echo To see general test output run the following from `pwd`:
+    echo $CALL_MOCHA "$ERRORCASE_FILES"
+    echo ======================================================================
+    exit 1
+}
+
+echo Error case results matched expected.

test/acceptance/error/exclude-pattern.txt

@@ -0,0 +1 @@
+\[".*",[{]"title":".*should not run.*",

test/acceptance/error/match-pattern.txt

@@ -0,0 +1,4 @@
+\["start"
+\["fail",[{]"title":".*should fail.*",
+\["pass",[{]"title":".*should pass.*",
+\["end",[{]"suites":4,"tests":8,"passes":4,"pending":0,"failures":4,

test/acceptance/error/timeout.js

@@ -0,0 +1,61 @@
+
+describe('timeouts', function(){
+  it('should fail on timeout', function(done){
+    setTimeout(done, 3000);
+  })
+
+  it('should pass test A after timeout', function(done){
+    setTimeout(done, 1000);
+  })
+  it('should pass run test B after timeout', function(done){
+    setTimeout(done, 1000);
+  })
+
+  it('should fail on timeout again', function(done){
+    setTimeout(done, 3000);
+  })
+})
+
+describe('timeouts', function(){
+
+  describe('timeouts', function(){
+    it('should fail on timeout', function(done){
+      setTimeout(done, 3000);
+    })
+
+    it('should pass run test A after timeout', function(done){
+      setTimeout(done, 1000);
+    })
+  })
+
+  describe('timeouts', function(){
+    it('should fail on timeout', function(done){
+      setTimeout(done, 3000);
+    })
+
+    it('should pass test A after timeout', function(done){
+      setTimeout(done, 1000);
+    })
+  })
+
+
+})
+
+
+// Known issue: #270
+// This will currently fail and stop further execution of mocha entirely.
+// describe('timeouts-beforeEach', function(){
+//   var waitTime = 2500;
+//   beforeEach(function(done){
+//     setTimeout(done, waitTime);
+//     waitTime -= 1000;
+//   })
+//
+//   it('should error on timeout during before each', function(done){
+//
+//   })
+//
+//   it('should pass without on timeout during before each', function(done){
+//
+//   })
+// })

test/acceptance/timeout.js

@@ -1,21 +1,11 @@
 
 describe('timeouts', function(){
-  beforeEach(function(done){
-    // uncomment
-    // setTimeout(done, 3000);
-    done();
-  })
-
-  it('should error on timeout', function(done){
-    // uncomment
-    // setTimeout(done, 3000);
-    done();
-  })
 
   it('should allow overriding per-test', function(done){
     this.timeout(1000);
     setTimeout(function(){
       done();
     }, 300);
   })
+
 })