Improve correspondence with Python errors and console behavior

Compiler and command-line error messages now reflect their Python counterparts.
E.g. where Python emits a `SyntaxError`, so does Hy; same for `TypeError`s.
Multiple tests have been added that check the format and type of raised
exceptions over varying command-line invocations (e.g. interactive and not).

A new exception type for `require` errors was added so that they can be treated
like normal run-time errors and not compiler errors.

The Hy REPL has been further refactored to better match the class-structured
API.  Now, different error types are handled separately and leverage more base
class-provided functionality.

Closes hylang#1486.

hy/_compat.py

@@ -43,6 +43,10 @@ def reraise(exc_type, value, traceback=None):
         finally:
             traceback = None
 
+    code_obj_args = ['argcount', 'kwonlyargcount', 'nlocals', 'stacksize',
+                     'flags', 'code', 'consts', 'names', 'varnames',
+                     'filename', 'name', 'firstlineno', 'lnotab', 'freevars',
+                     'cellvars']
 else:
     def raise_from(value, from_value=None):
         raise value
@@ -55,10 +59,30 @@ def reraise(exc_type, value, traceback=None):
             traceback = None
     ''')
 
+    code_obj_args = ['argcount', 'nlocals', 'stacksize', 'flags', 'code',
+                     'consts', 'names', 'varnames', 'filename', 'name',
+                     'firstlineno', 'lnotab', 'freevars', 'cellvars']
+
 raise_code = compile(raise_src, __file__, 'exec')
 exec(raise_code)
 
 
+def rename_function(func, new_name):
+    """Creates a copy of a function and [re]sets the name at the code-object
+    level.
+    """
+    c = func.__code__
+    new_code = type(c)(*[getattr(c, 'co_{}'.format(a))
+                         if a != 'name' else str(new_name)
+                         for a in code_obj_args])
+
+    _fn = type(func)(new_code, func.__globals__, str(new_name),
+                     func.__defaults__, func.__closure__)
+    _fn.__dict__.update(func.__dict__)
+
+    return _fn
+
+
 def isidentifier(x):
     if x in ('True', 'False', 'None', 'print'):
         # `print` is special-cased here because Python 2's

hy/cmdline.py

@@ -12,6 +12,7 @@
 import io
 import importlib
 import py_compile
+import traceback
 import runpy
 import types
 
@@ -20,8 +21,9 @@
 import hy
 from hy.lex import hy_parse, mangle
 from hy.lex.exceptions import PrematureEndOfInput
-from hy.compiler import HyASTCompiler, hy_compile, hy_eval
-from hy.errors import HySyntaxError, filtered_hy_exceptions
+from hy.compiler import HyASTCompiler, hy_eval, hy_compile, ast_compile
+from hy.errors import (HyLanguageError, HyRequireError, HyMacroExpansionError,
+                       filtered_hy_exceptions, hy_exc_handler)
 from hy.importer import runhy
 from hy.completer import completion, Completer
 from hy.macros import macro, require
@@ -50,53 +52,98 @@ def __call__(self, code=None):
 builtins.exit = HyQuitter('exit')
 
 
+class HyCommandCompiler(object):
+    def __init__(self, module, ast_callback=None, hy_compiler=None):
+        self.module = module
+        self.ast_callback = ast_callback
+        self.hy_compiler = hy_compiler
+
+    def __call__(self, source, filename="<input>", symbol="single"):
+        try:
+            hy_ast = hy_parse(source, filename=filename)
+            root_ast = ast.Interactive if symbol == 'single' else ast.Module
+
+            # Our compiler doesn't correspond to a real, fixed source file, so
+            # we need to [re]set these.
+            self.hy_compiler.filename = filename
+            self.hy_compiler.source = source
+            exec_ast, eval_ast = hy_compile(hy_ast, self.module, root=root_ast,
+                                            get_expr=True,
+                                            compiler=self.hy_compiler,
+                                            filename=filename, source=source)
+
+            if self.ast_callback:
+                self.ast_callback(exec_ast, eval_ast)
+
+            exec_code = ast_compile(exec_ast, filename, symbol)
+            eval_code = ast_compile(eval_ast, filename, 'eval')
+
+            return exec_code, eval_code
+        except PrematureEndOfInput:
+            # Save these so that we can reraise/display when an incomplete
+            # interactive command is given at the prompt.
+            sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+            return None
+
+
 class HyREPL(code.InteractiveConsole, object):
     def __init__(self, spy=False, output_fn=None, locals=None,
-                 filename="<input>"):
+                 filename="<stdin>"):
 
+        # Create a proper module for this REPL so that we can obtain it easily
+        # (e.g. using `importlib.import_module`).
+        # We let `InteractiveConsole` initialize `self.locals` when it's
+        # `None`.
         super(HyREPL, self).__init__(locals=locals,
                                      filename=filename)
 
-        # Create a proper module for this REPL so that we can obtain it easily
-        # (e.g. using `importlib.import_module`).
-        # Also, make sure it's properly introduced to `sys.modules` and
-        # consistently use its namespace as `locals` from here on.
         module_name = self.locals.get('__name__', '__console__')
+        # Make sure our newly created module is properly introduced to
+        # `sys.modules`, and consistently use its namespace as `self.locals`
+        # from here on.
         self.module = sys.modules.setdefault(module_name,
                                              types.ModuleType(module_name))
         self.module.__dict__.update(self.locals)
         self.locals = self.module.__dict__
 
         # Load cmdline-specific macros.
-        require('hy.cmdline', module_name, assignments='ALL')
+        require('hy.cmdline', self.module, assignments='ALL')
 
         self.hy_compiler = HyASTCompiler(self.module)
 
+        self.compile = HyCommandCompiler(self.module, self.ast_callback,
+                                         self.hy_compiler)
+
         self.spy = spy
+        self.last_value = None
 
         if output_fn is None:
             self.output_fn = repr
         elif callable(output_fn):
             self.output_fn = output_fn
+        elif "." in output_fn:
+            parts = [mangle(x) for x in output_fn.split(".")]
+            module, f = '.'.join(parts[:-1]), parts[-1]
+            self.output_fn = getattr(importlib.import_module(module), f)
         else:
-            if "." in output_fn:
-                parts = [mangle(x) for x in output_fn.split(".")]
-                module, f = '.'.join(parts[:-1]), parts[-1]
-                self.output_fn = getattr(importlib.import_module(module), f)
-            else:
-                self.output_fn = __builtins__[mangle(output_fn)]
+            self.output_fn = __builtins__[mangle(output_fn)]
 
         # Pre-mangle symbols for repl recent results: *1, *2, *3
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
 
-    def ast_callback(self, main_ast, expr_ast):
+    def ast_callback(self, exec_ast, eval_ast):
         if self.spy:
-            # Mush the two AST chunks into a single module for
-            # conversion into Python.
-            new_ast = ast.Module(main_ast.body +
-                                 [ast.Expr(expr_ast.body)])
-            print(astor.to_source(new_ast))
+            try:
+                # Mush the two AST chunks into a single module for
+                # conversion into Python.
+                new_ast = ast.Module(exec_ast.body +
+                                     [ast.Expr(eval_ast.body)])
+                print(astor.to_source(new_ast))
+            except Exception:
+                msg = 'Exception in AST callback:\n{}\n'.format(
+                    traceback.format_exc())
+                self.write(msg)
 
     def _error_wrap(self, error_fn, *args, **kwargs):
         sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
@@ -120,46 +167,49 @@ def showsyntaxerror(self, filename=None):
     def showtraceback(self):
         self._error_wrap(super(HyREPL, self).showtraceback)
 
-    def runsource(self, source, filename='<input>', symbol='single'):
-
+    def runcode(self, code):
         try:
-            do = hy_parse(source, filename=filename)
-        except PrematureEndOfInput:
-            return True
-        except HySyntaxError as e:
-            self.showsyntaxerror(filename=filename)
-            return False
-
-        try:
-            # Our compiler doesn't correspond to a real, fixed source file, so
-            # we need to [re]set these.
-            self.hy_compiler.filename = filename
-            self.hy_compiler.source = source
-            value = hy_eval(do, self.locals, self.module, self.ast_callback,
-                            compiler=self.hy_compiler, filename=filename,
-                            source=source)
+            eval(code[0], self.locals)
+            self.last_value = eval(code[1], self.locals)
+            self.print_last_value = True
         except SystemExit:
             raise
         except Exception as e:
+            # Set this to avoid a print-out of the last value on errors.
+            self.print_last_value = False
+            self.showtraceback()
+
+    def runsource(self, source, filename='<stdin>', symbol='exec'):
+        try:
+            res = super(HyREPL, self).runsource(source, filename, symbol)
+        except (HyMacroExpansionError, HyRequireError):
+            # We need to handle these exceptions ourselves, because the base
+            # method only handles `OverflowError`, `SyntaxError` and
+            # `ValueError`.
+            self.showsyntaxerror(filename)
+            return False
+        except (HyLanguageError):
+            # Our compiler will also raise `TypeError`s
             self.showtraceback()
             return False
 
-        if value is not None:
-            # Shift exisitng REPL results
-            next_result = value
+        # Shift exisitng REPL results
+        if not res:
+            next_result = self.last_value
             for sym in self._repl_results_symbols:
                 self.locals[sym], next_result = next_result, self.locals[sym]
 
             # Print the value.
-            try:
-                output = self.output_fn(value)
-            except Exception:
-                self.showtraceback()
-                return False
+            if self.print_last_value:
+                try:
+                    output = self.output_fn(self.last_value)
+                except Exception:
+                    self.showtraceback()
+                    return False
 
-            print(output)
+                print(output)
 
-        return False
+        return res
 
 
 @macro("koan")
@@ -215,9 +265,14 @@ def ideas_macro(ETname):
 
 
 def run_command(source, filename=None):
-    tree = hy_parse(source, filename=filename)
     __main__ = importlib.import_module('__main__')
     require("hy.cmdline", __main__, assignments="ALL")
+    try:
+        tree = hy_parse(source, filename=filename)
+    except HyLanguageError:
+        hy_exc_handler(*sys.exc_info())
+        return 1
+
     with filtered_hy_exceptions():
         hy_eval(tree, None, __main__, filename=filename, source=source)
     return 0
@@ -259,12 +314,18 @@ def run_icommand(source, **kwargs):
             source = f.read()
         filename = source
     else:
-        filename = '<input>'
+        filename = '<string>'
 
+    hr = HyREPL(**kwargs)
     with filtered_hy_exceptions():
-        hr = HyREPL(**kwargs)
-        hr.runsource(source, filename=filename, symbol='single')
-        return run_repl(hr)
+        res = hr.runsource(source, filename=filename)
+
+    # If the command was prematurely ended, show an error (just like Python
+    # does).
+    if res:
+        hy_exc_handler(sys.last_type, sys.last_value, sys.last_traceback)
+
+    return run_repl(hr)
 
 
 USAGE = "%(prog)s [-h | -i cmd | -c cmd | -m module | file | -] [arg] ..."
@@ -352,6 +413,7 @@ def cmdline_handler(scriptname, argv):
             return run_command(sys.stdin.read(), filename='<stdin>')
 
         else:
+
             # User did "hy <filename>"
             filename = options.args[0]
 
@@ -371,6 +433,9 @@ def cmdline_handler(scriptname, argv):
                 print("hy: Can't open file '{0}': [Errno {1}] {2}".format(
                       e.filename, e.errno, e.strerror), file=sys.stderr)
                 sys.exit(e.errno)
+            except HyLanguageError:
+                hy_exc_handler(*sys.exc_info())
+                sys.exit(1)
 
     # User did NOTHING!
     return run_repl(spy=options.spy, output_fn=options.repl_output_fn)
@@ -440,14 +505,15 @@ def hy2py_main():
     options = parser.parse_args(sys.argv[1:])
 
     if options.FILE is None or options.FILE == '-':
+        filename = '<stdin>'
         source = sys.stdin.read()
-        with filtered_hy_exceptions():
-            hst = hy_parse(source, filename='<stdin>')
     else:
-        with filtered_hy_exceptions(), \
-             io.open(options.FILE, 'r', encoding='utf-8') as source_file:
+        filename = options.FILE
+        with io.open(options.FILE, 'r', encoding='utf-8') as source_file:
             source = source_file.read()
-            hst = hy_parse(source, filename=options.FILE)
+
+    with filtered_hy_exceptions():
+        hst = hy_parse(source, filename=filename)
 
     if options.with_source:
         # need special printing on Windows in case the
@@ -464,7 +530,7 @@ def hy2py_main():
         print()
 
     with filtered_hy_exceptions():
-        _ast = hy_compile(hst, '__main__')
+        _ast = hy_compile(hst, '__main__', filename=filename, source=source)
 
     if options.with_ast:
         if PY3 and platform.system() == "Windows":

hy/compiler.py

@@ -9,8 +9,8 @@
 from hy.model_patterns import (FORM, SYM, KEYWORD, STR, sym, brackets, whole,
                                notpexpr, dolike, pexpr, times, Tag, tag, unpack)
 from funcparserlib.parser import some, many, oneplus, maybe, NoParseError
-from hy.errors import (HyCompileError, HyTypeError, HyEvalError,
-                       HyInternalError)
+from hy.errors import (HyCompileError, HyTypeError, HyLanguageError,
+                       HySyntaxError, HyEvalError, HyInternalError)
 
 from hy.lex import mangle, unmangle
 
@@ -443,15 +443,18 @@ def compile(self, tree):
             # nested; so let's re-raise this exception, let's not wrap it in
             # another HyCompileError!
             raise
-        except HyTypeError as e:
-            reraise(type(e), e, None)
+        except HyLanguageError as e:
+            # These are expected errors that should be passed to the user.
+            reraise(type(e), e, sys.exc_info()[2])
         except Exception as e:
+            # These are unexpected errors that will--hopefully--never be seen
+            # by the user.
             f_exc = traceback.format_exc()
             exc_msg = "Internal Compiler Bug 😱\n⤷ {}".format(f_exc)
             reraise(HyCompileError, HyCompileError(exc_msg), sys.exc_info()[2])
 
     def _syntax_error(self, message, expr):
-        return HyTypeError(message, self.filename, expr, self.source)
+        return HySyntaxError(message, expr, self.filename, self.source)
 
     def _compile_collect(self, exprs, with_kwargs=False, dict_display=False,
                          oldpy_unpack=False):