Cache command line source for exceptions

Source entered interactively can now be displayed in traceback output.  Also,
the REPL object is now available in its namespace, so that, for instance,
display options--like `spy`--can be turned on and off interactively.

hy/_compat.py

@@ -31,10 +31,7 @@
 if PY3:
     raise_src = textwrap.dedent('''
     def raise_from(value, from_value):
-        try:
-            raise value from from_value
-        finally:
-            traceback = None
+        raise value from from_value
     ''')
 
     def reraise(exc_type, value, traceback=None):

hy/cmdline.py

@@ -14,23 +14,33 @@
 import py_compile
 import traceback
 import runpy
+import time
+import linecache
+import hashlib
+import codeop
 
 import astor.code_gen
 
 import hy
 
+from contextlib import contextmanager
 from hy.lex import mangle
 from hy.lex.exceptions import PrematureEndOfInput
 from hy.errors import (HyLanguageError, HyRequireError, HyMacroExpansionError,
                        filtered_hy_exceptions, hy_exc_handler)
 from hy.compiler import hy_compile
-from hy.importer import hy_eval, hy_parse, runhy, ast_compile
+from hy.importer import hy_eval, hy_parse, runhy, hy_ast_compile_flags
 from hy.completer import completion, Completer
 from hy.macros import macro, require
 from hy.models import HyExpression, HyString, HySymbol
 from hy._compat import builtins, PY3, FileNotFoundError
 
 
+sys.last_type = None
+sys.last_value = None
+sys.last_traceback = None
+
+
 class HyQuitter(object):
     def __init__(self, name):
         self.name = name
@@ -51,31 +61,112 @@ def __call__(self, code=None):
 builtins.quit = HyQuitter('quit')
 builtins.exit = HyQuitter('exit')
 
+@contextmanager
+def extend_linecache(add_cmdline_cache):
+    _linecache_checkcache = linecache.checkcache
+
+    def _cmdline_checkcache(*args):
+        _linecache_checkcache(*args)
+        linecache.cache.update(add_cmdline_cache)
+
+    linecache.checkcache = _cmdline_checkcache
+    yield
+    linecache.checkcache = _linecache_checkcache
 
-class HyCommandCompiler(object):
-    def __init__(self, module_name='__console__', ast_callback=None):
+
+class HyCompile(codeop.Compile, object):
+    """This compiler uses `linecache` like `IPython.core.compilerop.CachingCompiler`."""
+
+    def __init__(self, module_name, locals, ast_callback=None,
+                 cmdline_cache={}):
         self.module_name = module_name
+        self.locals = locals
         self.ast_callback = ast_callback
 
+        super(HyCompile, self).__init__()
+
+        self.flags |= hy_ast_compile_flags
+
+        self.cmdline_cache = cmdline_cache
+
+    def _cache(self, source, name):
+        entry = (len(source),
+                 time.time(),
+                 [line + '\n' for line in source.splitlines()],
+                 name)
+
+        linecache.cache[name] = entry
+        self.cmdline_cache[name] = entry
+
+    def _update_exc_info(self):
+            self.locals['_hy_last_type'] = sys.last_type
+            self.locals['_hy_last_value'] = sys.last_value
+            # Skip our frame.
+            sys.last_traceback = getattr(sys.last_traceback, 'tb_next',
+                                         sys.last_traceback)
+            self.locals['_hy_last_traceback'] = sys.last_traceback
+
     def __call__(self, source, filename="<input>", symbol="single"):
+        # `codeop._maybe_compile` can return `source="pass"`.
+        if source == 'pass':
+            return None
+
+        hash_digest = hashlib.sha1(source.encode("utf-8").strip()).hexdigest()
+        name = '{}-{}'.format(filename.strip('<>'), hash_digest)
+
+        try:
+            hy_ast = hy_parse(source, filename=name)
+        except Exception:
+            # Capture a traceback without the compiler/REPL frames.
+            sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+            self._update_exc_info()
+            raise
+
+        self._cache(source, name)
+
+        root_ast = ast.Interactive if symbol == 'single' else ast.Module
+
         try:
-            hy_ast = hy_parse(source, filename=filename)
-            root_ast = ast.Interactive if symbol == 'single' else ast.Module
             exec_ast, eval_ast = hy_compile(hy_ast, self.module_name, root_ast,
-                                            get_expr=True, filename=filename,
+                                            get_expr=True, filename=name,
                                             source=source)
 
             if self.ast_callback:
                 self.ast_callback(exec_ast, eval_ast)
 
-            exec_code = ast_compile(exec_ast, filename, symbol)
-            eval_code = ast_compile(eval_ast, filename, 'eval')
+            exec_code = super(HyCompile, self).__call__(exec_ast, name, symbol)
+            eval_code = super(HyCompile, self).__call__(eval_ast, name, 'eval')
 
-            return exec_code, eval_code
-        except PrematureEndOfInput:
-            # Save these so that we can reraise/display when an incomplete
-            # interactive command is given at the prompt.
+        except HyLanguageError:
+            # Hy will raise exceptions during compile-time that Python would
+            # raise during run-time (e.g. import errors for `require`).  In
+            # order to work gracefully with the Python world, we convert such
+            # Hy errors to code that purposefully reraises those exceptions in
+            # the places where Python code expects them.
             sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
+            self._update_exc_info()
+            exec_code = super(HyCompile, self).__call__(
+                'import hy._compat; hy._compat.reraise('
+                '_hy_last_type, _hy_last_value, _hy_last_traceback)',
+                name, symbol)
+            eval_code = super(HyCompile, self).__call__('None', name, 'eval')
+
+        return exec_code, eval_code
+
+
+class HyCommandCompiler(codeop.CommandCompiler, object):
+    def __init__(self, *args, **kwargs):
+        self.compiler = HyCompile(*args, **kwargs)
+
+    def __call__(self, *args, **kwargs):
+        try:
+            return super(HyCommandCompiler, self).__call__(*args, **kwargs)
+        except PrematureEndOfInput:
+            # We have to do this here, because `codeop._maybe_compile` won't
+            # take `None` for a return value (at least not in Python 2.7) and
+            # this exception type is also a `SyntaxError`, so it will be caught
+            # by `code.InteractiveConsole` base methods before it reaches our
+            # `runsource`.
             return None
 
 
@@ -86,10 +177,16 @@ def __init__(self, spy=False, output_fn=None, locals=None,
         super(HyREPL, self).__init__(locals=locals, filename=filename)
 
         self.module_name = self.locals.get('__name__', '__console__')
-        self.compile = HyCommandCompiler(self.module_name, self.ast_callback)
+
+        self.cmdline_cache = {}
+        self.compile = HyCommandCompiler(self.module_name,
+                                         self.locals,
+                                         self.ast_callback,
+                                         self.cmdline_cache)
 
         self.spy = spy
         self.last_value = None
+        self.print_last_value = True
 
         if output_fn is None:
             self.output_fn = repr
@@ -106,6 +203,9 @@ def __init__(self, spy=False, output_fn=None, locals=None,
         self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)]
         self.locals.update({sym: None for sym in self._repl_results_symbols})
 
+        # Allow access to the running REPL instance
+        self.locals['_hy_repl'] = self
+
     def ast_callback(self, exec_ast, eval_ast):
         if self.spy:
             try:
@@ -119,11 +219,17 @@ def ast_callback(self, exec_ast, eval_ast):
                     traceback.format_exc())
                 self.write(msg)
 
-    def _error_wrap(self, error_fn, *args, **kwargs):
+    def _error_wrap(self, error_fn, exc_info_override=False, *args, **kwargs):
         sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info()
 
-        # Sadly, this method in Python 2.7 ignores an overridden
-        # `sys.excepthook`.
+        if exc_info_override:
+            # Use a traceback that doesn't have the REPL frames.
+            sys.last_type = self.locals.get('_hy_last_type', sys.last_type)
+            sys.last_value = self.locals.get('_hy_last_value', sys.last_value)
+            sys.last_traceback = self.locals.get('_hy_last_traceback',
+                                                 sys.last_traceback)
+
+        # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`.
         if sys.excepthook is sys.__excepthook__:
             error_fn(*args, **kwargs)
         else:
@@ -136,6 +242,7 @@ def showsyntaxerror(self, filename=None):
             filename = self.filename
 
         self._error_wrap(super(HyREPL, self).showsyntaxerror,
+                         exc_info_override=True,
                          filename=filename)
 
     def showtraceback(self):
@@ -261,10 +368,12 @@ def run_repl(hr=None, **kwargs):
 
     namespace = {'__name__': '__console__', '__doc__': ''}
 
-    with filtered_hy_exceptions(), completion(Completer(namespace)):
+    if not hr:
+        hr = HyREPL(locals=namespace, **kwargs)
 
-        if not hr:
-            hr = HyREPL(locals=namespace, **kwargs)
+    with filtered_hy_exceptions(), \
+         extend_linecache(hr.cmdline_cache), \
+         completion(Completer(namespace)):
 
         hr.interact("{appname} {version} using "
                     "{py}({build}) {pyversion} on {os}".format(

hy/errors.py

@@ -257,8 +257,7 @@ def hy_exc_filter(exc_type, exc_value, exc_traceback):
 
     lines = traceback.format_list(new_tb)
 
-    if lines:
-        lines.insert(0, "Traceback (most recent call last):\n")
+    lines.insert(0, "Traceback (most recent call last):\n")
 
     lines.extend(traceback.format_exception_only(exc_type, exc_value))
     output = ''.join(lines)

tests/test_bin.py

@@ -6,7 +6,6 @@
 
 import os
 import re
-import sys
 import shlex
 import subprocess
 
@@ -496,7 +495,7 @@ def test_bin_hy_tracebacks():
     assert output.startswith('=> ')
     error_lines = error.splitlines()
     if PY3:
-        output = error_lines[3]
+        output = error_lines[2]
         expected = "hy.errors.HyRequireError: No module named 'not-a-real-module'"
     else:
         output = error_lines[-3]
@@ -511,14 +510,19 @@ def test_bin_hy_tracebacks():
     #   SyntaxError: EOL while scanning string literal
     _, error = run_cmd('hy -c "(print \\""', expect=1)
     error_lines = error.splitlines()
-    expected = ['  File "<string>", line 1',
+    expected = ['Traceback (most recent call last):',
+                '  File "<string>", line 1',
                 '    (print "',
                 '           ^']
     if PY3:
         expected += ['hy.lex.exceptions.PrematureEndOfInput: Partial string literal']
     else:
         expected += ['PrematureEndOfInput: Partial string literal']
-    assert error_lines == expected
+
+    assert error_lines[0] == expected[0]
+    assert (error_lines[1] == expected[1] or
+            re.match('  File "string-[0-9a-f]+", line 1', error_lines[1]))
+    assert error_lines[2:] == expected[2:]
 
     # Modeled after
     #   > python -i -c "print('"
@@ -528,10 +532,13 @@ def test_bin_hy_tracebacks():
     #   SyntaxError: EOL while scanning string literal
     #   >>>
     output, error = run_cmd('hy -i "(print \\""')
-    assert output.startswith('=> ')
+    assert output.splitlines()[-1].startswith('=> ')
     error_lines = error.splitlines()
-    assert error_lines[:3] == expected[:-1:1]
-    assert error_lines[3].endswith(expected[-1])
+    assert error_lines[0] == expected[0]
+    assert (error_lines[1] == expected[1] or
+            re.match('  File "string-[0-9a-f]+", line 1', error_lines[1]))
+    assert error_lines[2:4] == expected[2:4]
+    assert error_lines[4].endswith(expected[-1])
 
     # Modeled after
     #   > python -c 'print(a)'