Generate more informative syntax errors

This commit refactors the exception/error classes and their handling, keeps Hy
source strings and their originating file information (if any) closer to the
origin of an exception (so that calling code isn't responsible for annotating
exceptions), and provides minimally intrusive traceback print-out filtering via
a context manager that temporarily alters `sys.excepthook` (enabled by default
for the Hy interpreter).

It also provides an environment variable, `HY_COLORED_ERRORS`, and package
variable, `hy.errors.__colored_errors`, that enables/disables manual error
coloring.

Closes #657, closes #1510, closes #1429.

hy/_compat.py

@@ -6,7 +6,10 @@
     import __builtin__ as builtins
 except ImportError:
     import builtins  # NOQA
-import sys, keyword
+
+import sys
+import keyword
+import textwrap
 
 PY3 = sys.version_info[0] >= 3
 PY35 = sys.version_info >= (3, 5)
@@ -23,10 +26,21 @@
 string_types = str   if PY3 else basestring   # NOQA
 
 if PY3:
-    exec('def raise_empty(t, *args): raise t(*args) from None')
+    reraise_src = textwrap.dedent('''
+    def reraise(exc_type, values, traceback=None):
+        if not isinstance(values, tuple):
+            values = values.args
+        raise exc_type(*values) from traceback
+    ''')
 else:
-    def raise_empty(t, *args):
-        raise t(*args)
+    reraise_src = textwrap.dedent('''
+    def reraise(exc_type, values, traceback=None):
+        raise exc_type, getattr(values, "args", values), traceback
+    ''')
+
+reraise_code = compile(reraise_src, __file__, 'exec')
+exec(reraise_code)
+
 
 def isidentifier(x):
     if x in ('True', 'False', 'None', 'print'):

hy/cmdline.py

@@ -17,8 +17,11 @@
 import astor.code_gen
 
 import hy
-from hy.lex import LexException, PrematureEndOfInput, mangle
-from hy.compiler import HyTypeError, hy_compile
+
+from hy.lex import mangle
+from hy.lex.exceptions import HySyntaxError, PrematureEndOfInput
+from hy.errors import filtered_hy_exceptions
+from hy.compiler import hy_compile
 from hy.importer import hy_eval, hy_parse, runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
@@ -47,7 +50,7 @@ def __call__(self, code=None):
 builtins.exit = HyQuitter('exit')
 
 
-class HyREPL(code.InteractiveConsole):
+class HyREPL(code.InteractiveConsole, object):
     def __init__(self, spy=False, output_fn=None, locals=None,
                  filename="<input>"):
 
@@ -65,33 +68,48 @@ def __init__(self, spy=False, output_fn=None, locals=None,
             else:
                 self.output_fn = __builtins__[mangle(output_fn)]
 
-        code.InteractiveConsole.__init__(self, locals=locals,
-                                         filename=filename)
+        super(HyREPL, self).__init__(locals=locals, filename=filename)
 
         # Pre-mangle symbols for repl recent results: *1, *2, *3
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
 
-    def runsource(self, source, filename='<input>', symbol='single'):
-        global SIMPLE_TRACEBACKS
+    def error_handler(self, e):
+        self.locals[mangle("*e")] = e
+        self.showtraceback()
 
-        def error_handler(e, use_simple_traceback=False):
-            self.locals[mangle("*e")] = e
-            if use_simple_traceback:
-                print(e, file=sys.stderr)
-            else:
-                self.showtraceback()
+    def showsyntaxerror(self, filename=None):
+        if filename is None:
+            filename = self.filename
+
+        sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+
+        # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`.
+        if sys.excepthook is sys.__excepthook__:
+            super(HyREPL, self).showsyntaxerror(filename=filename)
+        else:
+            sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback)
+
+        self.locals[mangle("*e")] = sys.last_value
 
+    def showtraceback(self):
+        sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+
+        # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`.
+        if sys.excepthook is sys.__excepthook__:
+            super(HyREPL, self).showtraceback()
+        else:
+            sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback)
+
+        self.locals[mangle("*e")] = sys.last_value
+
+    def runsource(self, source, filename='<input>', symbol='single'):
         try:
-            try:
-                do = hy_parse(source)
-            except PrematureEndOfInput:
-                return True
-        except LexException as e:
-            if e.source is None:
-                e.source = source
-                e.filename = filename
-            error_handler(e, use_simple_traceback=True)
+            do = hy_parse(source, filename=filename)
+        except PrematureEndOfInput:
+            return True
+        except HySyntaxError:
+            self.showsyntaxerror(filename=filename)
             return False
 
         try:
@@ -102,16 +120,13 @@ def ast_callback(main_ast, expr_ast):
                     new_ast = ast.Module(main_ast.body +
                                          [ast.Expr(expr_ast.body)])
                     print(astor.to_source(new_ast))
+
             value = hy_eval(do, self.locals, "__console__",
-                            ast_callback)
-        except HyTypeError as e:
-            if e.source is None:
-                e.source = source
-                e.filename = filename
-            error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS)
-            return False
-        except Exception as e:
-            error_handler(e)
+                            ast_callback, filename=filename, source=source)
+        except SystemExit:
+            raise
+        except Exception:
+            self.showtraceback()
             return False
 
         if value is not None:
@@ -123,10 +138,12 @@ def ast_callback(main_ast, expr_ast):
             # Print the value.
             try:
                 output = self.output_fn(value)
-            except Exception as e:
-                error_handler(e)
+            except Exception:
+                self.showtraceback()
                 return False
+
             print(output)
+
         return False
 
 
@@ -184,22 +201,11 @@ def ideas_macro(ETname):
 require("hy.cmdline", "__console__", assignments="ALL")
 require("hy.cmdline", "__main__", assignments="ALL")
 
-SIMPLE_TRACEBACKS = True
-
 
-def pretty_error(func, *args, **kw):
-    try:
-        return func(*args, **kw)
-    except (HyTypeError, LexException) as e:
-        if SIMPLE_TRACEBACKS:
-            print(e, file=sys.stderr)
-            sys.exit(1)
-        raise
-
-
-def run_command(source):
-    tree = hy_parse(source)
-    pretty_error(hy_eval, tree, module_name="__main__")
+def run_command(source, filename=None):
+    tree = hy_parse(source, filename=filename)
+    with filtered_hy_exceptions():
+        hy_eval(tree, module_name="__main__", filename=filename, source=source)
     return 0
 
 
@@ -210,7 +216,7 @@ def run_repl(hr=None, **kwargs):
 
     namespace = {'__name__': '__console__', '__doc__': ''}
 
-    with completion(Completer(namespace)):
+    with filtered_hy_exceptions(), completion(Completer(namespace)):
 
         if not hr:
             hr = HyREPL(locals=namespace, **kwargs)
@@ -243,9 +249,10 @@ def run_icommand(source, **kwargs):
     else:
         filename = '<input>'
 
-    hr = HyREPL(**kwargs)
-    hr.runsource(source, filename=filename, symbol='single')
-    return run_repl(hr)
+    with filtered_hy_exceptions():
+        hr = HyREPL(**kwargs)
+        hr.runsource(source, filename=filename, symbol='single')
+        return run_repl(hr)
 
 
 USAGE = "%(prog)s [-h | -i cmd | -c cmd | -m module | file | -] [arg] ..."
@@ -281,9 +288,6 @@ def cmdline_handler(scriptname, argv):
                              "(e.g., hy.contrib.hy-repr.hy-repr)")
     parser.add_argument("-v", "--version", action="version", version=VERSION)
 
-    parser.add_argument("--show-tracebacks", action="store_true",
-                        help="show complete tracebacks for Hy exceptions")
-
     # this will contain the script/program name and any arguments for it.
     parser.add_argument('args', nargs=argparse.REMAINDER,
                         help=argparse.SUPPRESS)
@@ -308,10 +312,6 @@ def cmdline_handler(scriptname, argv):
 
     options = parser.parse_args(argv[1:])
 
-    if options.show_tracebacks:
-        global SIMPLE_TRACEBACKS
-        SIMPLE_TRACEBACKS = False
-
     if options.E:
         # User did "hy -E ..."
         _remove_python_envs()
@@ -321,7 +321,7 @@ def cmdline_handler(scriptname, argv):
 
     if options.command:
         # User did "hy -c ..."
-        return run_command(options.command)
+        return run_command(options.command, filename='<string>')
 
     if options.mod:
         # User did "hy -m ..."
@@ -337,7 +337,7 @@ def cmdline_handler(scriptname, argv):
     if options.args:
         if options.args[0] == "-":
             # Read the program from stdin
-            return run_command(sys.stdin.read())
+            return run_command(sys.stdin.read(), filename='<stdin>')
 
         else:
             # User did "hy <filename>"
@@ -352,7 +352,8 @@ def cmdline_handler(scriptname, argv):
 
             try:
                 sys.argv = options.args
-                runhy.run_path(filename, run_name='__main__')
+                with filtered_hy_exceptions():
+                    runhy.run_path(filename, run_name='__main__')
                 return 0
             except FileNotFoundError as e:
                 print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
@@ -427,41 +428,44 @@ def hy2py_main():
 
     options = parser.parse_args(sys.argv[1:])
 
-    if options.FILE is None or options.FILE == '-':
-        source = sys.stdin.read()
-    else:
-        with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
-            source = source_file.read()
-
-    hst = pretty_error(hy_parse, source)
-    if options.with_source:
-        # need special printing on Windows in case the
-        # codepage doesn't support utf-8 characters
-        if PY3 and platform.system() == "Windows":
-            for h in hst:
-                try:
-                    print(h)
-                except:
-                    print(str(h).encode('utf-8'))
-        else:
-            print(hst)
-        print()
-        print()
-
-    _ast = pretty_error(hy_compile, hst, module_name)
-    if options.with_ast:
-        if PY3 and platform.system() == "Windows":
-            _print_for_windows(astor.dump_tree(_ast))
+    with filtered_hy_exceptions():
+        if options.FILE is None or options.FILE == '-':
+            source = sys.stdin.read()
+            hst = hy_parse(source, filename='<stdin>')
         else:
-            print(astor.dump_tree(_ast))
-        print()
-        print()
+            with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
+                source = source_file.read()
+            hst = hy_parse(source, filename=options.FILE)
+
+        if options.with_source:
+            # need special printing on Windows in case the
+            # codepage doesn't support utf-8 characters
+            if PY3 and platform.system() == "Windows":
+                for h in hst:
+                    try:
+                        print(h)
+                    except Exception:
+                        print(str(h).encode('utf-8'))
+            else:
+                print(hst)
+            print()
+            print()
 
-    if not options.without_python:
-        if PY3 and platform.system() == "Windows":
-            _print_for_windows(astor.code_gen.to_source(_ast))
-        else:
-            print(astor.code_gen.to_source(_ast))
+        _ast = hy_compile(hst, module_name)
+
+        if options.with_ast:
+            if PY3 and platform.system() == "Windows":
+                _print_for_windows(astor.dump_tree(_ast))
+            else:
+                print(astor.dump_tree(_ast))
+            print()
+            print()
+
+        if not options.without_python:
+            if PY3 and platform.system() == "Windows":
+                _print_for_windows(astor.code_gen.to_source(_ast))
+            else:
+                print(astor.code_gen.to_source(_ast))
 
     parser.exit(0)