From 79c54e17eadacb012bdcf8cb25e7efebebc9e8e3 Mon Sep 17 00:00:00 2001 From: "Brandon T. Willard" Date: Wed, 10 Oct 2018 01:46:49 -0500 Subject: [PATCH] Generate more informative syntax errors This commit refactors the exception/error classes and their handling, keeps Hy source strings and their originating file information (if any) closer to the origin of an exception (so that calling code isn't responsible for annotating exceptions), and provides minimally intrusive traceback print-out filtering via a context manager that temporarily alters `sys.excepthook` (enabled by default for the Hy interpreter). It also provides an environment variable, `HY_COLORED_ERRORS`, and package variable, `hy.errors.__colored_errors`, that enables/disables manual error coloring. Closes hylang/hy#657, closes hylang/hy#1510, closes hylang/hy#1429. --- hy/_compat.py | 22 ++- hy/cmdline.py | 194 ++++++++++++++------------- hy/compiler.py | 159 ++++++++++++---------- hy/core/bootstrap.hy | 16 +-- hy/core/language.hy | 3 +- hy/errors.py | 229 +++++++++++++++++++++++++------- hy/importer.py | 102 +++++++------- hy/lex/__init__.py | 40 ++++-- hy/lex/exceptions.py | 47 +------ hy/lex/parser.py | 130 +++++++++++------- hy/macros.py | 17 ++- tests/compilers/test_ast.py | 35 +++-- tests/importer/test_importer.py | 6 +- tests/test_bin.py | 2 +- tests/test_lex.py | 102 +++++++++++++- 15 files changed, 700 insertions(+), 404 deletions(-) diff --git a/hy/_compat.py b/hy/_compat.py index 4416ea5b6..1680212c6 100644 --- a/hy/_compat.py +++ b/hy/_compat.py @@ -6,7 +6,10 @@ import __builtin__ as builtins except ImportError: import builtins # NOQA -import sys, keyword + +import sys +import keyword +import textwrap PY3 = sys.version_info[0] >= 3 PY35 = sys.version_info >= (3, 5) @@ -23,10 +26,21 @@ string_types = str if PY3 else basestring # NOQA if PY3: - exec('def raise_empty(t, *args): raise t(*args) from None') + reraise_src = textwrap.dedent(''' + def reraise(exc_type, values, traceback=None): + if not isinstance(values, tuple): + values = values.args + raise exc_type(*values) from traceback + ''') else: - def raise_empty(t, *args): - raise t(*args) + reraise_src = textwrap.dedent(''' + def reraise(exc_type, values, traceback=None): + raise exc_type, getattr(values, "args", values), traceback + ''') + +reraise_code = compile(reraise_src, __file__, 'exec') +exec(reraise_code) + def isidentifier(x): if x in ('True', 'False', 'None', 'print'): diff --git a/hy/cmdline.py b/hy/cmdline.py index f38355bd9..d77735937 100644 --- a/hy/cmdline.py +++ b/hy/cmdline.py @@ -17,8 +17,11 @@ import astor.code_gen import hy -from hy.lex import LexException, PrematureEndOfInput, mangle -from hy.compiler import HyTypeError, hy_compile + +from hy.lex import mangle +from hy.lex.exceptions import HySyntaxError, PrematureEndOfInput +from hy.errors import filtered_hy_exceptions +from hy.compiler import hy_compile from hy.importer import hy_eval, hy_parse, runhy from hy.completer import completion, Completer from hy.macros import macro, require @@ -47,7 +50,7 @@ def __call__(self, code=None): builtins.exit = HyQuitter('exit') -class HyREPL(code.InteractiveConsole): +class HyREPL(code.InteractiveConsole, object): def __init__(self, spy=False, output_fn=None, locals=None, filename=""): @@ -65,33 +68,48 @@ def __init__(self, spy=False, output_fn=None, locals=None, else: self.output_fn = __builtins__[mangle(output_fn)] - code.InteractiveConsole.__init__(self, locals=locals, - filename=filename) + super(HyREPL, self).__init__(locals=locals, filename=filename) # Pre-mangle symbols for repl recent results: *1, *2, *3 self._repl_results_symbols = [mangle("*{}".format(i + 1)) for i in range(3)] self.locals.update({sym: None for sym in self._repl_results_symbols}) - def runsource(self, source, filename='', symbol='single'): - global SIMPLE_TRACEBACKS + def error_handler(self, e): + self.locals[mangle("*e")] = e + self.showtraceback() - def error_handler(e, use_simple_traceback=False): - self.locals[mangle("*e")] = e - if use_simple_traceback: - print(e, file=sys.stderr) - else: - self.showtraceback() + def showsyntaxerror(self, filename=None): + if filename is None: + filename = self.filename + + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + + # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`. + if sys.excepthook is sys.__excepthook__: + super(HyREPL, self).showsyntaxerror(filename=filename) + else: + sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback) + + self.locals[mangle("*e")] = sys.last_value + def showtraceback(self): + sys.last_type, sys.last_value, sys.last_traceback = sys.exc_info() + + # Sadly, this method in Python 2.7 ignores an overridden `sys.excepthook`. + if sys.excepthook is sys.__excepthook__: + super(HyREPL, self).showtraceback() + else: + sys.excepthook(sys.last_type, sys.last_value, sys.last_traceback) + + self.locals[mangle("*e")] = sys.last_value + + def runsource(self, source, filename='', symbol='single'): try: - try: - do = hy_parse(source) - except PrematureEndOfInput: - return True - except LexException as e: - if e.source is None: - e.source = source - e.filename = filename - error_handler(e, use_simple_traceback=True) + do = hy_parse(source, filename=filename) + except PrematureEndOfInput: + return True + except HySyntaxError: + self.showsyntaxerror(filename=filename) return False try: @@ -102,16 +120,13 @@ def ast_callback(main_ast, expr_ast): new_ast = ast.Module(main_ast.body + [ast.Expr(expr_ast.body)]) print(astor.to_source(new_ast)) + value = hy_eval(do, self.locals, "__console__", - ast_callback) - except HyTypeError as e: - if e.source is None: - e.source = source - e.filename = filename - error_handler(e, use_simple_traceback=SIMPLE_TRACEBACKS) - return False - except Exception as e: - error_handler(e) + ast_callback, filename=filename, source=source) + except SystemExit: + raise + except Exception: + self.showtraceback() return False if value is not None: @@ -123,10 +138,12 @@ def ast_callback(main_ast, expr_ast): # Print the value. try: output = self.output_fn(value) - except Exception as e: - error_handler(e) + except Exception: + self.showtraceback() return False + print(output) + return False @@ -184,22 +201,11 @@ def ideas_macro(ETname): require("hy.cmdline", "__console__", assignments="ALL") require("hy.cmdline", "__main__", assignments="ALL") -SIMPLE_TRACEBACKS = True - -def pretty_error(func, *args, **kw): - try: - return func(*args, **kw) - except (HyTypeError, LexException) as e: - if SIMPLE_TRACEBACKS: - print(e, file=sys.stderr) - sys.exit(1) - raise - - -def run_command(source): - tree = hy_parse(source) - pretty_error(hy_eval, tree, module_name="__main__") +def run_command(source, filename=None): + tree = hy_parse(source, filename=filename) + with filtered_hy_exceptions(): + hy_eval(tree, module_name="__main__", filename=filename, source=source) return 0 @@ -210,7 +216,7 @@ def run_repl(hr=None, **kwargs): namespace = {'__name__': '__console__', '__doc__': ''} - with completion(Completer(namespace)): + with filtered_hy_exceptions(), completion(Completer(namespace)): if not hr: hr = HyREPL(locals=namespace, **kwargs) @@ -243,9 +249,10 @@ def run_icommand(source, **kwargs): else: filename = '' - hr = HyREPL(**kwargs) - hr.runsource(source, filename=filename, symbol='single') - return run_repl(hr) + with filtered_hy_exceptions(): + hr = HyREPL(**kwargs) + hr.runsource(source, filename=filename, symbol='single') + return run_repl(hr) USAGE = "%(prog)s [-h | -i cmd | -c cmd | -m module | file | -] [arg] ..." @@ -281,9 +288,6 @@ def cmdline_handler(scriptname, argv): "(e.g., hy.contrib.hy-repr.hy-repr)") parser.add_argument("-v", "--version", action="version", version=VERSION) - parser.add_argument("--show-tracebacks", action="store_true", - help="show complete tracebacks for Hy exceptions") - # this will contain the script/program name and any arguments for it. parser.add_argument('args', nargs=argparse.REMAINDER, help=argparse.SUPPRESS) @@ -308,10 +312,6 @@ def cmdline_handler(scriptname, argv): options = parser.parse_args(argv[1:]) - if options.show_tracebacks: - global SIMPLE_TRACEBACKS - SIMPLE_TRACEBACKS = False - if options.E: # User did "hy -E ..." _remove_python_envs() @@ -321,7 +321,7 @@ def cmdline_handler(scriptname, argv): if options.command: # User did "hy -c ..." - return run_command(options.command) + return run_command(options.command, filename='') if options.mod: # User did "hy -m ..." @@ -337,7 +337,7 @@ def cmdline_handler(scriptname, argv): if options.args: if options.args[0] == "-": # Read the program from stdin - return run_command(sys.stdin.read()) + return run_command(sys.stdin.read(), filename='') else: # User did "hy " @@ -352,7 +352,8 @@ def cmdline_handler(scriptname, argv): try: sys.argv = options.args - runhy.run_path(filename, run_name='__main__') + with filtered_hy_exceptions(): + runhy.run_path(filename, run_name='__main__') return 0 except FileNotFoundError as e: print("hy: Can't open file '{0}': [Errno {1}] {2}".format( @@ -427,41 +428,44 @@ def hy2py_main(): options = parser.parse_args(sys.argv[1:]) - if options.FILE is None or options.FILE == '-': - source = sys.stdin.read() - else: - with io.open(options.FILE, 'r', encoding='utf-8') as source_file: - source = source_file.read() - - hst = pretty_error(hy_parse, source) - if options.with_source: - # need special printing on Windows in case the - # codepage doesn't support utf-8 characters - if PY3 and platform.system() == "Windows": - for h in hst: - try: - print(h) - except: - print(str(h).encode('utf-8')) - else: - print(hst) - print() - print() - - _ast = pretty_error(hy_compile, hst, module_name) - if options.with_ast: - if PY3 and platform.system() == "Windows": - _print_for_windows(astor.dump_tree(_ast)) + with filtered_hy_exceptions(): + if options.FILE is None or options.FILE == '-': + source = sys.stdin.read() + hst = hy_parse(source, filename='') else: - print(astor.dump_tree(_ast)) - print() - print() + with io.open(options.FILE, 'r', encoding='utf-8') as source_file: + source = source_file.read() + hst = hy_parse(source, filename=options.FILE) + + if options.with_source: + # need special printing on Windows in case the + # codepage doesn't support utf-8 characters + if PY3 and platform.system() == "Windows": + for h in hst: + try: + print(h) + except Exception: + print(str(h).encode('utf-8')) + else: + print(hst) + print() + print() - if not options.without_python: - if PY3 and platform.system() == "Windows": - _print_for_windows(astor.code_gen.to_source(_ast)) - else: - print(astor.code_gen.to_source(_ast)) + _ast = hy_compile(hst, module_name) + + if options.with_ast: + if PY3 and platform.system() == "Windows": + _print_for_windows(astor.dump_tree(_ast)) + else: + print(astor.dump_tree(_ast)) + print() + print() + + if not options.without_python: + if PY3 and platform.system() == "Windows": + _print_for_windows(astor.code_gen.to_source(_ast)) + else: + print(astor.code_gen.to_source(_ast)) parser.exit(0) diff --git a/hy/compiler.py b/hy/compiler.py index bd43cf7f2..356583ec4 100755 --- a/hy/compiler.py +++ b/hy/compiler.py @@ -14,8 +14,7 @@ from hy.lex import mangle, unmangle import hy.macros -from hy._compat import ( - str_type, bytes_type, long_type, PY3, PY35, raise_empty) +from hy._compat import (str_type, bytes_type, long_type, PY3, PY35, reraise) from hy.macros import require, macroexpand, tag_macroexpand import hy.importer @@ -284,10 +283,16 @@ def is_unpack(kind, x): class HyASTCompiler(object): - def __init__(self, module_name): + + + def __init__(self, module_name, filename='', source=None): self.anon_var_count = 0 self.imports = defaultdict(set) self.module_name = module_name + + self.filename = filename + self.source = source + self.temp_if = None self.can_use_stdlib = ( not module_name.startswith("hy.core") @@ -355,10 +360,12 @@ def compile(self, tree): # nested; so let's re-raise this exception, let's not wrap it in # another HyCompileError! raise - except HyTypeError: - raise + except HyTypeError as e: + reraise(type(e), e, None) except Exception as e: - raise_empty(HyCompileError, e, sys.exc_info()[2]) + f_exc = traceback.format_exc() + exc_msg = "Internal Compiler Bug 😱\n⤷ {}".format(''.join(f_exc)) + reraise(HyCompileError, (exc_msg,), sys.exc_info()[2]) def _compile_collect(self, exprs, with_kwargs=False, dict_display=False, oldpy_unpack=False): @@ -379,8 +386,9 @@ def _compile_collect(self, exprs, with_kwargs=False, dict_display=False, if not PY35 and oldpy_unpack and is_unpack("iterable", expr): if oldpy_starargs: - raise HyTypeError(expr, "Pythons < 3.5 allow only one " - "`unpack-iterable` per call") + raise HyTypeError("Pythons < 3.5 allow only one " + "`unpack-iterable` per call", + (self.filename, expr, self.source)) oldpy_starargs = self.compile(expr[1]) ret += oldpy_starargs oldpy_starargs = oldpy_starargs.force_expr @@ -396,21 +404,23 @@ def _compile_collect(self, exprs, with_kwargs=False, dict_display=False, expr, arg=None, value=ret.force_expr)) elif oldpy_unpack: if oldpy_kwargs: - raise HyTypeError(expr, "Pythons < 3.5 allow only one " - "`unpack-mapping` per call") + raise HyTypeError("Pythons < 3.5 allow only one " + "`unpack-mapping` per call", + (self.filename, expr, self.source)) oldpy_kwargs = ret.force_expr elif with_kwargs and isinstance(expr, HyKeyword): try: value = next(exprs_iter) except StopIteration: - raise HyTypeError(expr, - "Keyword argument {kw} needs " - "a value.".format(kw=expr)) + raise HyTypeError("Keyword argument {kw} needs " + "a value.".format(kw=expr), + (self.filename, expr, self.source)) if not expr: - raise HyTypeError(expr, "Can't call a function with the " - "empty keyword") + raise HyTypeError("Can't call a function with the " + "empty keyword", + (self.filename, expr, self.source)) compiled_value = self.compile(value) ret += compiled_value @@ -451,8 +461,8 @@ def _storeize(self, expr, name, func=None): if isinstance(name, Result): if not name.is_expr(): - raise HyTypeError(expr, - "Can't assign or delete a non-expression") + raise HyTypeError("Can't assign or delete a non-expression", + (self.filename, expr, self.source)) name = name.expr if isinstance(name, (ast.Tuple, ast.List)): @@ -471,9 +481,8 @@ def _storeize(self, expr, name, func=None): new_name = ast.Starred( value=self._storeize(expr, name.value, func)) else: - raise HyTypeError(expr, - "Can't assign or delete a %s" % - type(expr).__name__) + raise HyTypeError("Can't assign or delete a %s" % type(expr).__name__, + (self.filename, expr, self.source)) new_name.ctx = func() ast.copy_location(new_name, name) @@ -499,9 +508,8 @@ def _render_quoted_form(self, form, level): op = unmangle(ast_str(form[0])) if level == 0 and op in ("unquote", "unquote-splice"): if len(form) != 2: - raise HyTypeError(form, - ("`%s' needs 1 argument, got %s" % - op, len(form) - 1)) + raise HyTypeError("`%s' needs 1 argument, got %s" % op, len(form) - 1, + (self.filename, expr, self.source)) return set(), form[1], op == "unquote-splice" elif op == "quasiquote": level += 1 @@ -553,7 +561,8 @@ def compile_quote(self, expr, root, arg): @special("unpack-iterable", [FORM]) def compile_unpack_iterable(self, expr, root, arg): if not PY3: - raise HyTypeError(expr, "`unpack-iterable` isn't allowed here") + raise HyTypeError("`unpack-iterable` isn't allowed here", + (self.filename, expr, self.source)) ret = self.compile(arg) ret += asty.Starred(expr, value=ret.force_expr, ctx=ast.Load()) return ret @@ -583,7 +592,8 @@ def compile_raise_expression(self, expr, root, exc, cause): if cause is not None: if not PY3: - raise HyTypeError(expr, "raise from only supported in python 3") + raise HyTypeError("raise from only supported in python 3", + (self.filename, expr, self.source)) cause = self.compile(cause) ret += cause cause = cause.force_expr @@ -630,14 +640,12 @@ def compile_try_expression(self, expr, root, body, catchers, orelse, finalbody): # Using (else) without (except) is verboten! if orelse and not handlers: - raise HyTypeError( - expr, - "`try' cannot have `else' without `except'") + raise HyTypeError("`try' cannot have `else' without `except'", + (self.filename, expr, self.source)) # Likewise a bare (try) or (try BODY). if not (handlers or finalbody): - raise HyTypeError( - expr, - "`try' must have an `except' or `finally' clause") + raise HyTypeError("`try' must have an `except' or `finally' clause", + (self.filename, expr, self.source)) returnable = Result( expr=asty.Name(expr, id=return_var.id, ctx=ast.Load()), @@ -887,7 +895,8 @@ def c(e): def compile_decorate_expression(self, expr, name, args): decs, fn = args[:-1], self.compile(args[-1]) if not fn.stmts or not isinstance(fn.stmts[-1], _decoratables): - raise HyTypeError(args[-1], "Decorated a non-function") + raise HyTypeError("Decorated a non-function", + (self.filename, args[-1], self.source)) decs, ret, _ = self._compile_collect(decs) fn.stmts[-1].decorator_list = decs + fn.stmts[-1].decorator_list return ret + fn @@ -1123,8 +1132,9 @@ def compile_import_or_require(self, expr, root, entries): if (HySymbol('*'), None) in kids: if len(kids) != 1: star = kids[kids.index((HySymbol('*'), None))][0] - raise HyTypeError(star, "* in an import name list " - "must be on its own") + raise HyTypeError("* in an import name list " + "must be on its own", + (self.filename, star, self.source)) else: assignments = [(k, v or k) for k, v in kids] @@ -1305,15 +1315,15 @@ def _compile_assign(self, name, result): if str_name in (["None"] + (["True", "False"] if PY3 else [])): # Python 2 allows assigning to True and False, although # this is rarely wise. - raise HyTypeError(name, - "Can't assign to `%s'" % str_name) + raise HyTypeError("Can't assign to `%s'" % str_name, + (self.filename, name, self.source)) result = self.compile(result) ld_name = self.compile(name) if isinstance(ld_name.expr, ast.Call): - raise HyTypeError(name, - "Can't assign to a callable: `%s'" % str_name) + raise HyTypeError("Can't assign to a callable: `%s'" % str_name, + (self.filename, name, self.source)) if (result.temp_variables and isinstance(name, HySymbol) @@ -1389,7 +1399,8 @@ def compile_function_def(self, expr, root, params, body): mandatory, optional, rest, kwonly, kwargs = params optional, defaults, ret = self._parse_optional_args(optional) if kwonly is not None and not PY3: - raise HyTypeError(params, "&kwonly parameters require Python 3") + raise HyTypeError("&kwonly parameters require Python 3", + (self.filename, params, self.source)) kwonly, kw_defaults, ret2 = self._parse_optional_args(kwonly, True) ret += ret2 main_args = mandatory + optional @@ -1533,7 +1544,8 @@ def compile_eval_and_compile(self, expr, root, body): 'hy': hy, '__name__': self.module_name} hy.importer.hy_eval(new_expr + body, self._namespaces[self.module_name], - self.module_name) + self.module_name, + filename=self.filename) return (self._compile_branch(body) if ast_str(root) == "eval_and_compile" else Result()) @@ -1547,8 +1559,8 @@ def compile_expression(self, expr): return self.compile(expr) if not expr: - raise HyTypeError( - expr, "empty expressions are not allowed at top level") + raise HyTypeError("empty expressions are not allowed at top level", + (self.filename, expr, self.source)) args = list(expr) root = args.pop(0) @@ -1566,20 +1578,19 @@ def compile_expression(self, expr): sroot in (mangle(","), mangle(".")) or not any(is_unpack("iterable", x) for x in args)): if sroot in _bad_roots: - raise HyTypeError( - expr, - "The special form '{}' is not allowed here".format(root)) + raise HyTypeError("The special form '{}' is not allowed here".format(root), + (self.filename, expr, self.source)) # `sroot` is a special operator. Get the build method and # pattern-match the arguments. build_method, pattern = _special_form_compilers[sroot] try: parse_tree = pattern.parse(args) except NoParseError as e: - raise HyTypeError( - expr[min(e.state.pos + 1, len(expr) - 1)], - "parse error for special form '{}': {}".format( - root, - e.msg.replace("", "end of form"))) + raise HyTypeError("parse error for special form '{}': {}".format( + root, e.msg.replace("", "end of form")), + (self.filename, + expr[min(e.state.pos + 1, len(expr) - 1)], + self.source)) return Result() + build_method( self, expr, unmangle(sroot), *parse_tree) @@ -1601,13 +1612,13 @@ def compile_expression(self, expr): FORM + many(FORM)).parse(args) except NoParseError: - raise HyTypeError( - expr, "attribute access requires object") + raise HyTypeError("attribute access requires object", + (self.filename, expr, self.source)) # Reconstruct `args` to exclude `obj`. args = [x for p in kws for x in p] + list(rest) if is_unpack("iterable", obj): - raise HyTypeError( - obj, "can't call a method on an unpacking form") + raise HyTypeError("can't call a method on an unpacking form", + (self.filename, obj, self.source)) func = self.compile(HyExpression( [HySymbol(".").replace(root), obj] + attrs)) @@ -1645,16 +1656,17 @@ def compile_symbol(self, symbol): glob, local = symbol.rsplit(".", 1) if not glob: - raise HyTypeError(symbol, 'cannot access attribute on ' - 'anything other than a name ' - '(in order to get attributes of ' - 'expressions, use ' - '`(. {attr})` or ' - '`(.{attr} )`)'.format( - attr=local)) + raise HyTypeError('cannot access attribute on ' + 'anything other than a name ' + '(in order to get attributes of ' + 'expressions, use ' + '`(. {attr})` or ' + '`(.{attr} )`)'.format(attr=local), + (self.filename, symbol, self.source)) if not local: - raise HyTypeError(symbol, 'cannot access empty attribute') + raise HyTypeError('cannot access empty attribute', + (self.filename, symbol, self.source)) glob = HySymbol(glob).replace(symbol) ret = self.compile_symbol(glob) @@ -1699,7 +1711,8 @@ def compile_dict(self, m): return ret + asty.Dict(m, keys=keyvalues[::2], values=keyvalues[1::2]) -def hy_compile(tree, module_name, root=ast.Module, get_expr=False): +def hy_compile(tree, module_name, root=ast.Module, get_expr=False, + filename=None, source=None): """ Compile a HyObject tree into a Python AST Module. @@ -1709,10 +1722,22 @@ def hy_compile(tree, module_name, root=ast.Module, get_expr=False): tree = wrap_value(tree) if not isinstance(tree, HyObject): - raise HyCompileError("`tree` must be a HyObject or capable of " - "being promoted to one") - - compiler = HyASTCompiler(module_name) + raise TypeError("`tree` must be a HyObject or capable of " + "being promoted to one") + + try: + mod_loader = pkgutil.get_loader(module_name) + if filename is None: + filename = mod_loader.get_filename() + # TODO: Lazy load this. + if source is None: + source = mod_loader.get_source() + except Exception: + pass + + filename = filename or '' + + compiler = HyASTCompiler(module_name, filename=filename, source=source) result = compiler.compile(tree) expr = result.force_expr diff --git a/hy/core/bootstrap.hy b/hy/core/bootstrap.hy index cc576bc74..2255844dd 100644 --- a/hy/core/bootstrap.hy +++ b/hy/core/bootstrap.hy @@ -14,15 +14,15 @@ (if* (not (isinstance macro-name hy.models.HySymbol)) (raise (hy.errors.HyTypeError - macro-name (% "received a `%s' instead of a symbol for macro name" (. (type name) - __name__))))) + __name__)) + (, None macro-name None)))) (for [kw '[&kwonly &kwargs]] (if* (in kw lambda-list) - (raise (hy.errors.HyTypeError macro-name - (% "macros cannot use %s" - kw))))) + (raise (hy.errors.HyTypeError (% "macros cannot use %s" + kw) + (, None macro-name None))))) ;; this looks familiar... `(eval-and-compile (import hy) @@ -45,9 +45,9 @@ (if (and (not (isinstance tag-name hy.models.HySymbol)) (not (isinstance tag-name hy.models.HyString))) (raise (hy.errors.HyTypeError - tag-name (% "received a `%s' instead of a symbol for tag macro name" - (. (type tag-name) __name__))))) + (. (type tag-name) __name__)) + (, None tag-name None)))) (if (or (= tag-name ":") (= tag-name "&")) (raise (NameError (% "%s can't be used as a tag macro name" tag-name)))) @@ -60,7 +60,7 @@ (defmacro macro-error [location reason] "Error out properly within a macro at `location` giving `reason`." - `(raise (hy.errors.HyMacroExpansionError ~location ~reason))) + `(raise (hy.errors.HyMacroExpansionError ~reason (, None ~location None)))) (defmacro defn [name lambda-list &rest body] "Define `name` as a function with `lambda-list` signature and body `body`." diff --git a/hy/core/language.hy b/hy/core/language.hy index abb387fc6..5ea13cfa1 100644 --- a/hy/core/language.hy +++ b/hy/core/language.hy @@ -19,7 +19,8 @@ (import [collections :as cabc]) (import [collections.abc :as cabc])) (import [hy.models [HySymbol HyKeyword]]) -(import [hy.lex [LexException PrematureEndOfInput tokenize mangle unmangle]]) +(import [hy.lex [tokenize mangle unmangle]]) +(import [hy.lex.exceptions [PrematureEndOfInput]]) (import [hy.compiler [HyASTCompiler]]) (import [hy.importer [hy-eval :as eval]]) diff --git a/hy/errors.py b/hy/errors.py index a60b09cb5..a0ae0ca0a 100644 --- a/hy/errors.py +++ b/hy/errors.py @@ -2,52 +2,88 @@ # Copyright 2018 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. - +import os +import sys import traceback +import pkgutil + +from contextlib import contextmanager + + +__colored_errors = False -from clint.textui import colored + +def colored_errors(): + global __colored_errors + return os.environ.get('HY_COLORED_ERRORS', __colored_errors) class HyError(Exception): + def __init__(self, message, *args): + self.message = message + super(HyError, self).__init__(message, *args) + + +class HyInternalError(HyError): + """Unexpected errors occurring during compilation or parsing of Hy code. + + Errors sub-classing this are not intended to be user-facing. """ - Generic Hy error. All internal Exceptions will be subclassed from this - Exception. + + def __init__(self, message, *args): + super(HyInternalError, self).__init__(message, *args) + + +class HyLanguageError(HyError): + """Errors occurring during the expected use of Hy. + + Errors sub-classing this will be user-facing. """ - pass + def __init__(self, message, *args): + super(HyLanguageError, self).__init__(message, *args) -class HyCompileError(HyError): - def __init__(self, exception, traceback=None): - self.exception = exception - self.traceback = traceback - def __str__(self): - if isinstance(self.exception, HyTypeError): - return str(self.exception) - if self.traceback: - tb = "".join(traceback.format_tb(self.traceback)).strip() - else: - tb = "No traceback available. 😟" - return("Internal Compiler Bug 😱\n⤷ %s: %s\nCompilation traceback:\n%s" - % (self.exception.__class__.__name__, - self.exception, tb)) +class HyCompileError(HyInternalError): + def __init__(self, message, *args): + super(HyCompileError, self).__init__(message, *args) -class HyTypeError(TypeError): - def __init__(self, expression, message): - super(HyTypeError, self).__init__(message) - self.expression = expression +class HyTypeError(HyLanguageError, TypeError): + def __init__(self, message, args=None): + """ + Parameters + ---------- + message: str + The exception's message. + args: tuple, optional + A tuple with the form `(filename, expression, source)`. + """ self.message = message - self.source = None - self.filename = None + + if args: + self.filename, self.expression, self.source = args + else: + self.filename = self.expression = self.source = None + + new_args = (self.filename, self.expression, self.source) + + super(HyTypeError, self).__init__(message, new_args) def __str__(self): result = "" - if all(getattr(self.expression, x, None) is not None - for x in ("start_line", "start_column", "end_column")): + if colored_errors(): + from clint.textui import colored + red, green, yellow = colored.red, colored.green, colored.yellow + else: + red = green = yellow = lambda x: x + has_syntax_info = all(hasattr(self.expression, x) + for x in ("start_line", "start_column", + "end_column")) + if has_syntax_info: line = self.expression.start_line start = self.expression.start_column end = self.expression.end_column @@ -61,44 +97,145 @@ def __str__(self): else: length = len(source[0]) - start - result += ' File "%s", line %d, column %d\n\n' % (self.filename, - line, - start) + result += ' File "%s", line %d, column %d\n\n' % (self.filename, line, start) if len(source) == 1: - result += ' %s\n' % colored.red(source[0]) + result += ' %s\n' % red(source[0]) result += ' %s%s\n' % (' '*(start-1), - colored.green('^' + '-'*(length-1) + '^')) + green('^' + '-'*(length-1) + '^')) if len(source) > 1: - result += ' %s\n' % colored.red(source[0]) + result += ' %s\n' % red(source[0]) result += ' %s%s\n' % (' '*(start-1), - colored.green('^' + '-'*length)) + green('^' + '-'*length)) if len(source) > 2: # write the middle lines for line in source[1:-1]: - result += ' %s\n' % colored.red("".join(line)) - result += ' %s\n' % colored.green("-"*len(line)) + result += ' %s\n' % red("".join(line)) + result += ' %s\n' % green("-"*len(line)) # write the last line - result += ' %s\n' % colored.red("".join(source[-1])) - result += ' %s\n' % colored.green('-'*(end-1) + '^') + result += ' %s\n' % red("".join(source[-1])) + result += ' %s\n' % green('-'*(end-1) + '^') else: result += ' File "%s", unknown location\n' % self.filename - result += colored.yellow("%s: %s\n\n" % - (self.__class__.__name__, - self.message)) + result += yellow("%s: %s\n\n" % (self.__class__.__name__, self.message)) return result class HyMacroExpansionError(HyTypeError): - pass + def __init__(self, message, args=None): + super(HyMacroExpansionError, self).__init__(message, args) -class HyIOError(HyError, IOError): +class HyIOError(HyInternalError, IOError): + """ Subclass used to distinguish between IOErrors raised by Hy itself as + opposed to Hy programs. """ - Trivial subclass of IOError and HyError, to distinguish between - IOErrors raised by Hy itself as opposed to Hy programs. + + def __init__(self, message, *args): + super(HyIOError, self).__init__(message, *args) + + +class HySyntaxError(HyLanguageError, SyntaxError): + """Error during the Lexing of a Hython expression.""" + + def __init__(self, message, args=None): + """ + Parameters + ---------- + message: str + The exception's message. + args: tuple, optional + A tuple with the form `(filename, lineno, colno, source)`. + """ + self.message = message + + if args: + self.filename, self.lineno, self.colno, self.source = args + else: + self.filename = self.lineno = self.colno = self.source = None + + new_args = (self.filename, self.lineno, self.colno, self.source) + + super(HySyntaxError, self).__init__(message, new_args) + + def __str__(self): + + output = traceback.format_exception_only(SyntaxError, + SyntaxError(*self.args)) + + if colored_errors(): + from hy.errors import colored + + output[-1] = colored.yellow(output[-1]) + if len(self.source) > 0: + output[-2] = colored.green(output[-2]) + for line in output[::-2]: + if line.strip().startswith( + 'File "{}", line'.format(self.filename)): + break + output[-3] = colored.red(output[-3]) + + return ''.join(output) + + +def _get_module_info(module): + compiler_loader = pkgutil.get_loader(module) + is_pkg = compiler_loader.is_package(module) + filename = compiler_loader.get_filename() + if is_pkg: + # Use package directory + return os.path.dirname(filename) + else: + # Normalize filename endings, because tracebacks will use `pyc` when + # the loader says `py`. + return filename.replace('.pyc', '.py') + + +__tb_hidden_modules = {_get_module_info(m) + for m in ['hy.compiler', 'hy.lex', + 'hy.cmdline', 'hy.lex.parser', + 'hy.importer', 'hy._compat', + 'rply']} + + +def hy_exc_handler(exc_type, exc_value, exc_traceback): + """Produce exceptions print-outs with all frames originating from the + modules in `__tb_hidden_modules` filtered out. + + The frames are actually filtered by each module's filename and only when a + subclass of `HyLanguageError` is emitted. + + This does not remove the frames from the actual tracebacks, so debugging + will show everything. """ - pass + + try: + # frame = (filename, line number, function name*, text) + new_tb = [frame for frame in traceback.extract_tb(exc_traceback) + if not (frame[0].replace('.pyc', '.py') in __tb_hidden_modules or + os.path.dirname(frame[0]) in __tb_hidden_modules)] + + lines = traceback.format_list(new_tb) + + if lines: + lines.insert(0, "Traceback (most recent call last):\n") + + lines.extend(traceback.format_exception_only(exc_type, exc_value)) + output = ''.join(lines) + + sys.stderr.write(output) + sys.stderr.flush() + except Exception: + sys.__excepthook__(exc_type, exc_value, exc_traceback) + + +@contextmanager +def filtered_hy_exceptions(): + """Remove Hy internal frames from tracebacks.""" + current_hook = sys.excepthook + sys.excepthook = hy_exc_handler + yield + sys.excepthook = current_hook diff --git a/hy/importer.py b/hy/importer.py index 1bc857377..ae01a77cf 100644 --- a/hy/importer.py +++ b/hy/importer.py @@ -6,7 +6,6 @@ import sys import os -import ast import inspect import pkgutil import re @@ -18,23 +17,23 @@ from functools import partial -from hy.errors import HyTypeError from hy.compiler import hy_compile -from hy.lex import tokenize, LexException +from hy.lex import tokenize from hy.models import HyExpression, HySymbol -from hy._compat import string_types, PY3 +from hy.lex.exceptions import HySyntaxError +from hy._compat import string_types, PY3, reraise hy_ast_compile_flags = (__future__.CO_FUTURE_DIVISION | __future__.CO_FUTURE_PRINT_FUNCTION) -def ast_compile(ast, filename, mode): +def ast_compile(hy_ast, filename, mode): """Compile AST. Parameters ---------- - ast : instance of `ast.AST` + hy_ast : instance of `ast.AST` filename : str Filename used for run-time error messages @@ -46,10 +45,10 @@ def ast_compile(ast, filename, mode): ------- out : instance of `types.CodeType` """ - return compile(ast, filename, mode, hy_ast_compile_flags) + return compile(hy_ast, filename, mode, hy_ast_compile_flags) -def hy_parse(source): +def hy_parse(source, filename=None): """Parse a Hy source string. Parameters @@ -57,21 +56,38 @@ def hy_parse(source): source: string Source code to parse. + filename: string, optional + File name corresponding to source. + Returns ------- - out : instance of `types.CodeType` + out : HyExpression """ - source = re.sub(r'\A#!.*', '', source) - return HyExpression([HySymbol("do")] + tokenize(source + "\n")) + _source = re.sub(r'\A#!.*', '', source) + # TODO: Could check for shebang removal a better way. + lineno_offset = 1 if _source != source else 0 + source = _source + try: + return HyExpression([HySymbol("do")] + + tokenize(source + "\n", + filename=filename, + lineno_offset=lineno_offset)) + except HySyntaxError as e: + reraise(type(e), e.args, None) -def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): +def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None, + filename=None, source=None): """Evaluates a quoted expression and returns the value. The optional second and third arguments specify the dictionary of globals to use and the module name. The globals dictionary defaults to ``(local)`` and the module name defaults to the name of the current module. + NOTE: If you're evaluating hand-crafted AST trees, make sure line numbers + are set properly. Try `fix_missing_locations` and related functions in the + Python `ast` library. + Examples -------- @@ -102,6 +118,15 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): expression object, in that order, after compilation but before evaluation. + filename: str, optional + The filename given to `compile`. Will be obtained from the module + given by `module_name` if `None` (and available). + + source: str, optional + Source string to use for error messages (e.g. macro expansion errors). + Will be obtained from the module given by `module_name` if `None` (and + available). + Returns ------- out : Result of evaluating the Hy compiled tree. @@ -117,16 +142,8 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): if not isinstance(module_name, string_types): raise TypeError("Module name must be a string") - _ast, expr = hy_compile(hytree, module_name, get_expr=True) - - # Spoof the positions in the generated ast... - for node in ast.walk(_ast): - node.lineno = 1 - node.col_offset = 1 - - for node in ast.walk(expr): - node.lineno = 1 - node.col_offset = 1 + _ast, expr = hy_compile(hytree, module_name, get_expr=True, + filename=filename, source=source) if ast_callback: ast_callback(_ast, expr) @@ -134,11 +151,13 @@ def hy_eval(hytree, namespace=None, module_name=None, ast_callback=None): if not isinstance(namespace, dict): raise TypeError("Globals must be a dictionary") + filename = filename or '' + # Two-step eval: eval() the body of the exec call - eval(ast_compile(_ast, "", "exec"), namespace) + eval(ast_compile(_ast, filename, "exec"), namespace) # Then eval the expression context and return that - return eval(ast_compile(expr, "", "eval"), namespace) + return eval(ast_compile(expr, filename, "eval"), namespace) def cache_from_source(source_path): @@ -224,14 +243,8 @@ def _could_be_hy_src(filename): def _hy_source_to_code(self, data, path, _optimize=-1): if _could_be_hy_src(path): source = data.decode("utf-8") - try: - hy_tree = hy_parse(source) - data = hy_compile(hy_tree, self.name) - except (HyTypeError, LexException) as e: - if e.source is None: - e.source = source - e.filename = path - raise + hy_tree = hy_parse(source, filename=path) + data = hy_compile(hy_tree, self.name) return _py_source_to_code(self, data, path, _optimize=_optimize) @@ -339,18 +352,14 @@ def byte_compile_hy(self, fullname=None): fullname = self._fix_name(fullname) if fullname is None: fullname = self.fullname - try: - hy_source = self.get_source(fullname) - hy_tree = hy_parse(hy_source) - hy_ast = hy_compile(hy_tree, fullname) - - code = compile(hy_ast, self.filename, 'exec', - hy_ast_compile_flags) - except (HyTypeError, LexException) as e: - if e.source is None: - e.source = hy_source - e.filename = self.filename - raise + + hy_source = self.get_source(fullname) + + hy_tree = hy_parse(hy_source, filename=self.filename) + hy_ast = hy_compile(hy_tree, fullname) + + code = compile(hy_ast, self.filename, 'exec', + hy_ast_compile_flags) if not sys.dont_write_bytecode: try: @@ -500,15 +509,12 @@ def hyc_compile(file_or_code, cfile=None, dfile=None, doraise=False): try: flags = None if _could_be_hy_src(filename): - hy_tree = hy_parse(source_str) + hy_tree = hy_parse(source_str, filename=filename) source = hy_compile(hy_tree, '') flags = hy_ast_compile_flags codeobject = compile(source, dfile or filename, 'exec', flags) except Exception as err: - if isinstance(err, (HyTypeError, LexException)) and err.source is None: - err.source = source_str - err.filename = filename py_exc = py_compile.PyCompileError(err.__class__, err, dfile or filename) diff --git a/hy/lex/__init__.py b/hy/lex/__init__.py index 5c0514380..02a453f5a 100644 --- a/hy/lex/__init__.py +++ b/hy/lex/__init__.py @@ -4,27 +4,45 @@ from __future__ import unicode_literals -import re, unicodedata +import re +import unicodedata + from hy._compat import str_type, isidentifier, UCS4 -from hy.lex.exceptions import LexException, PrematureEndOfInput # NOQA +from hy.lex.exceptions import LexException # NOQA -def tokenize(buf): - """ - Tokenize a Lisp file or string buffer into internal Hy objects. + +class ParserState(object): + def __init__(self, source, filename, lineno_offset=0): + self.source = source + self.filename = filename + self.lineno_offset = lineno_offset + + +def tokenize(source, filename=None, lineno_offset=0): + """ Tokenize a Lisp file or string buffer into internal Hy objects. + + Parameters + ---------- + source: str + The source to tokenize. + filename: str, optional + The filename corresponding to `source`. + lineno_offset: int, optional + An offset for the line numbering. Useful when `source` has been + manipulated relative to `filename` (e.g. removal of shebang headers). """ from hy.lex.lexer import lexer from hy.lex.parser import parser from rply.errors import LexingError + try: - return parser.parse(lexer.lex(buf)) + return parser.parse(lexer.lex(source), + state=ParserState(source, filename, lineno_offset)) except LexingError as e: pos = e.getsourcepos() raise LexException("Could not identify the next token.", - pos.lineno, pos.colno, buf) - except LexException as e: - if e.source is None: - e.source = buf - raise + (filename, pos.lineno + lineno_offset, pos.colno, + source)) mangle_delim = 'X' diff --git a/hy/lex/exceptions.py b/hy/lex/exceptions.py index 14df78bfe..7feb741c2 100644 --- a/hy/lex/exceptions.py +++ b/hy/lex/exceptions.py @@ -1,49 +1,12 @@ # Copyright 2018 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. +from hy.errors import HySyntaxError -from hy.errors import HyError +class LexException(HySyntaxError): + pass -class LexException(HyError): - """Error during the Lexing of a Hython expression.""" - def __init__(self, message, lineno, colno, source=None): - super(LexException, self).__init__(message) - self.message = message - self.lineno = lineno - self.colno = colno - self.source = source - self.filename = '' - def __str__(self): - from hy.errors import colored - - line = self.lineno - start = self.colno - - result = "" - - source = self.source.split("\n") - - if line > 0 and start > 0: - result += ' File "%s", line %d, column %d\n\n' % (self.filename, - line, - start) - - if len(self.source) > 0: - source_line = source[line-1] - else: - source_line = "" - - result += ' %s\n' % colored.red(source_line) - result += ' %s%s\n' % (' '*(start-1), colored.green('^')) - - result += colored.yellow("LexException: %s\n\n" % self.message) - - return result - - -class PrematureEndOfInput(LexException): - """We got a premature end of input""" - def __init__(self, message): - super(PrematureEndOfInput, self).__init__(message, -1, -1) +class PrematureEndOfInput(HySyntaxError): + pass diff --git a/hy/lex/parser.py b/hy/lex/parser.py index a13c79380..8a4660868 100755 --- a/hy/lex/parser.py +++ b/hy/lex/parser.py @@ -22,84 +22,85 @@ def set_boundaries(fun): @wraps(fun) - def wrapped(p): + def wrapped(state, p): start = p[0].source_pos end = p[-1].source_pos - ret = fun(p) - ret.start_line = start.lineno + ret = fun(state, p) + ret.start_line = start.lineno + state.lineno_offset ret.start_column = start.colno if start is not end: ret.end_line = end.lineno ret.end_column = end.colno else: - ret.end_line = start.lineno + ret.end_line = start.lineno + state.lineno_offset ret.end_column = start.colno + len(p[0].value) + ret.end_line += state.lineno_offset return ret return wrapped def set_quote_boundaries(fun): @wraps(fun) - def wrapped(p): + def wrapped(state, p): start = p[0].source_pos - ret = fun(p) - ret.start_line = start.lineno + ret = fun(state, p) + ret.start_line = start.lineno + state.lineno_offset ret.start_column = start.colno - ret.end_line = p[-1].end_line + ret.end_line = p[-1].end_line + state.lineno_offset ret.end_column = p[-1].end_column return ret return wrapped @pg.production("main : list_contents") -def main(p): +def main(state, p): return p[0] @pg.production("main : $end") -def main_empty(p): +def main_empty(state, p): return [] -def reject_spurious_dots(*items): - "Reject the spurious dots from items" - for list in items: - for tok in list: - if tok == "." and type(tok) == HySymbol: - raise LexException("Malformed dotted list", - tok.start_line, tok.start_column) +# def reject_spurious_dots(*items): +# "Reject the spurious dots from items" +# for list in items: +# for tok in list: +# if tok == "." and type(tok) == HySymbol: +# raise LexException("Malformed dotted list", +# tok.start_line, tok.start_column) @pg.production("paren : LPAREN list_contents RPAREN") @set_boundaries -def paren(p): +def paren(state, p): return HyExpression(p[1]) @pg.production("paren : LPAREN RPAREN") @set_boundaries -def empty_paren(p): +def empty_paren(state, p): return HyExpression([]) @pg.production("list_contents : term list_contents") -def list_contents(p): +def list_contents(state, p): return [p[0]] + p[1] @pg.production("list_contents : term") -def list_contents_single(p): +def list_contents_single(state, p): return [p[0]] @pg.production("list_contents : DISCARD term discarded_list_contents") -def list_contents_empty(p): +def list_contents_empty(state, p): return [] @pg.production("discarded_list_contents : DISCARD term discarded_list_contents") @pg.production("discarded_list_contents :") -def discarded_list_contents(p): +def discarded_list_contents(state, p): pass @@ -109,42 +110,42 @@ def discarded_list_contents(p): @pg.production("term : list") @pg.production("term : set") @pg.production("term : string") -def term(p): +def term(state, p): return p[0] @pg.production("term : DISCARD term term") -def term_discard(p): +def term_discard(state, p): return p[2] @pg.production("term : QUOTE term") @set_quote_boundaries -def term_quote(p): +def term_quote(state, p): return HyExpression([HySymbol("quote"), p[1]]) @pg.production("term : QUASIQUOTE term") @set_quote_boundaries -def term_quasiquote(p): +def term_quasiquote(state, p): return HyExpression([HySymbol("quasiquote"), p[1]]) @pg.production("term : UNQUOTE term") @set_quote_boundaries -def term_unquote(p): +def term_unquote(state, p): return HyExpression([HySymbol("unquote"), p[1]]) @pg.production("term : UNQUOTESPLICE term") @set_quote_boundaries -def term_unquote_splice(p): +def term_unquote_splice(state, p): return HyExpression([HySymbol("unquote-splice"), p[1]]) @pg.production("term : HASHSTARS term") @set_quote_boundaries -def term_hashstars(p): +def term_hashstars(state, p): n_stars = len(p[0].getstr()[1:]) if n_stars == 1: sym = "unpack-iterable" @@ -154,13 +155,13 @@ def term_hashstars(p): raise LexException( "Too many stars in `#*` construct (if you want to unpack a symbol " "beginning with a star, separate it with whitespace)", - p[0].source_pos.lineno, p[0].source_pos.colno) + _get_exc_args(state, p[0])) return HyExpression([HySymbol(sym), p[1]]) @pg.production("term : HASHOTHER term") @set_quote_boundaries -def hash_other(p): +def hash_other(state, p): # p == [(Token('HASHOTHER', '#foo'), bar)] st = p[0].getstr()[1:] str_object = HyString(st) @@ -170,63 +171,64 @@ def hash_other(p): @pg.production("set : HLCURLY list_contents RCURLY") @set_boundaries -def t_set(p): +def t_set(state, p): return HySet(p[1]) @pg.production("set : HLCURLY RCURLY") @set_boundaries -def empty_set(p): +def empty_set(state, p): return HySet([]) @pg.production("dict : LCURLY list_contents RCURLY") @set_boundaries -def t_dict(p): +def t_dict(state, p): return HyDict(p[1]) @pg.production("dict : LCURLY RCURLY") @set_boundaries -def empty_dict(p): +def empty_dict(state, p): return HyDict([]) @pg.production("list : LBRACKET list_contents RBRACKET") @set_boundaries -def t_list(p): +def t_list(state, p): return HyList(p[1]) @pg.production("list : LBRACKET RBRACKET") @set_boundaries -def t_empty_list(p): +def t_empty_list(state, p): return HyList([]) @pg.production("string : STRING") @set_boundaries -def t_string(p): +def t_string(state, p): # Replace the single double quotes with triple double quotes to allow # embedded newlines. try: s = eval(p[0].value.replace('"', '"""', 1)[:-1] + '"""') except SyntaxError: raise LexException("Can't convert {} to a HyString".format(p[0].value), - p[0].source_pos.lineno, p[0].source_pos.colno) + _get_exc_args(state, p[0])) return (HyString if isinstance(s, str_type) else HyBytes)(s) @pg.production("string : PARTIAL_STRING") -def t_partial_string(p): +def t_partial_string(state, p): # Any unterminated string requires more input - raise PrematureEndOfInput("Premature end of input") + raise PrematureEndOfInput("Partial string literal", + _get_exc_args(state, p[0])) bracket_string_re = next(r.re for r in lexer.rules if r.name == 'BRACKETSTRING') @pg.production("string : BRACKETSTRING") @set_boundaries -def t_bracket_string(p): +def t_bracket_string(state, p): m = bracket_string_re.match(p[0].value) delim, content = m.groups() return HyString(content, brackets=delim) @@ -234,7 +236,7 @@ def t_bracket_string(p): @pg.production("identifier : IDENTIFIER") @set_boundaries -def t_identifier(p): +def t_identifier(state, p): obj = p[0].value val = symbol_like(obj) @@ -247,7 +249,7 @@ def t_identifier(p): 'Cannot access attribute on anything other than a name (in ' 'order to get attributes of expressions, use ' '`(. )` or `(. )`)', - p[0].source_pos.lineno, p[0].source_pos.colno) + _get_exc_args(state, p[0])) return HySymbol(obj) @@ -283,15 +285,47 @@ def symbol_like(obj): return HyKeyword(obj[1:]) +def _get_exc_args(state, token): + source_pos = token.getsourcepos() + if token.source_pos: + lineno = source_pos.lineno + state.lineno_offset + colno = source_pos.colno + else: + lineno = -1 + colno = -1 + + if state.source: + lines = state.source.splitlines() + if lines[-1] == '': + del lines[-1] + + if lineno < 1: + lineno = len(lines) + colno = len(lines[-1]) + + source = lines[lineno - 1] + return (state.filename, lineno, colno, source) + + @pg.error -def error_handler(token): +def error_handler(state, token): tokentype = token.gettokentype() if tokentype == '$end': - raise PrematureEndOfInput("Premature end of input") + source_pos = token.source_pos or token.getsourcepos() + source = state.source + if source_pos: + lineno = source_pos.lineno + state.lineno_offset + colno = source_pos.colno + else: + lineno = -1 + colno = -1 + + raise PrematureEndOfInput("Premature end of input", + _get_exc_args(state, token)) else: raise LexException( "Ran into a %s where it wasn't expected." % tokentype, - token.source_pos.lineno, token.source_pos.colno) + _get_exc_args(state, token)) parser = pg.build() diff --git a/hy/macros.py b/hy/macros.py index a86f04302..59707b804 100644 --- a/hy/macros.py +++ b/hy/macros.py @@ -10,7 +10,7 @@ import hy.inspect from hy.models import replace_hy_obj, HyExpression, HySymbol, wrap_value from hy.lex import mangle -from hy._compat import str_type +from hy._compat import str_type, reraise from hy.errors import HyTypeError, HyMacroExpansionError @@ -184,7 +184,9 @@ def macroexpand(tree, compiler, once=False): except TypeError as e: msg = "expanding `" + str(tree[0]) + "': " msg += str(e).replace("()", "", 1).strip() - raise HyMacroExpansionError(tree, msg) + reraise(HyMacroExpansionError, + (msg, (compiler.filename, tree, compiler.source)), + None) try: obj = m(compiler.module_name, *tree[1:], **opts) @@ -194,7 +196,10 @@ def macroexpand(tree, compiler, once=False): raise except Exception as e: msg = "expanding `" + str(tree[0]) + "': " + repr(e) - raise HyMacroExpansionError(tree, msg) + reraise(HyMacroExpansionError, + (msg, (compiler.filename, tree, compiler.source)), + None) + tree = replace_hy_obj(obj, tree) if once: @@ -203,6 +208,7 @@ def macroexpand(tree, compiler, once=False): tree = wrap_value(tree) return tree + def macroexpand_1(tree, compiler): """Expand the toplevel macro from `tree` once, in the context of `compiler`.""" @@ -219,9 +225,8 @@ def tag_macroexpand(tag, tree, compiler): tag_macro = _hy_tag[None][tag] except KeyError: raise HyTypeError( - tag, - "`{0}' is not a defined tag macro.".format(tag) - ) + "`{0}' is not a defined tag macro.".format(tag), + (None, tag, None)) expr = tag_macro(tree) return replace_hy_obj(expr, tree) diff --git a/tests/compilers/test_ast.py b/tests/compilers/test_ast.py index cb3ab7bbb..a69a7a7e8 100644 --- a/tests/compilers/test_ast.py +++ b/tests/compilers/test_ast.py @@ -8,8 +8,8 @@ from hy import HyString from hy.models import HyObject from hy.importer import hy_compile, hy_eval, hy_parse -from hy.errors import HyCompileError, HyTypeError -from hy.lex.exceptions import LexException +from hy.errors import HyCompileError, HyTypeError, HyError +from hy.lex.exceptions import LexException, PrematureEndOfInput from hy._compat import PY3 import ast @@ -34,21 +34,22 @@ def can_eval(expr): def cant_compile(expr): - try: + with pytest.raises(HyError) as excinfo: hy_compile(hy_parse(expr), "__main__") - assert False - except HyTypeError as e: + + if issubclass(excinfo.type, HyTypeError): # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. - assert isinstance(e.expression, HyObject) - assert e.message - return e - except HyCompileError as e: + assert isinstance(excinfo.value.expression, HyObject) + assert excinfo.value.message + return excinfo.value + elif issubclass(excinfo.type, HyCompileError): # Anything that can't be compiled should raise a user friendly # error, otherwise it's a compiler bug. - assert isinstance(e.exception, HyTypeError) - assert e.traceback - return e + assert 'Internal Compiler Bug' in excinfo.value.message + return excinfo.value + else: + assert False def s(x): @@ -59,11 +60,9 @@ def test_ast_bad_type(): "Make sure AST breakage can happen" class C: pass - try: + + with pytest.raises(TypeError): hy_compile(C(), "__main__") - assert True is False - except TypeError: - pass def test_empty_expr(): @@ -473,7 +472,7 @@ def test_lambda_list_keywords_kwonly(): else: exception = cant_compile(kwonly_demo) assert isinstance(exception, HyTypeError) - message, = exception.args + message, args = exception.args assert message == "&kwonly parameters require Python 3" @@ -546,7 +545,7 @@ def test_compile_error(): def test_for_compile_error(): """Ensure we get compile error in tricky 'for' cases""" - with pytest.raises(LexException) as excinfo: + with pytest.raises(PrematureEndOfInput) as excinfo: can_compile("(fn [] (for)") assert excinfo.value.message == "Premature end of input" diff --git a/tests/importer/test_importer.py b/tests/importer/test_importer.py index 900da0314..48515040b 100644 --- a/tests/importer/test_importer.py +++ b/tests/importer/test_importer.py @@ -17,7 +17,7 @@ import hy from hy._compat import bytes_type from hy.errors import HyTypeError -from hy.lex import LexException +from hy.lex.exceptions import PrematureEndOfInput from hy.compiler import hy_compile from hy.importer import hy_parse, HyLoader, cache_from_source @@ -186,14 +186,14 @@ def unlink(filename): assert mod.a == 11 assert mod.b == 20 - # Now cause a LexException + # Now cause a syntax error unlink(source) with open(source, "w") as f: f.write("(setv a 11") f.write("(setv b (// 20 1))") - with pytest.raises(LexException): + with pytest.raises(PrematureEndOfInput): imp.reload(mod) mod = sys.modules.get(TESTFN) diff --git a/tests/test_bin.py b/tests/test_bin.py index 58d144884..aaf75c137 100644 --- a/tests/test_bin.py +++ b/tests/test_bin.py @@ -149,7 +149,7 @@ def test_bin_hy_stdin_unlocatable_hytypeerror(): # inside run_cmd. _, err = run_cmd("hy", """ (import hy.errors) - (raise (hy.errors.HyTypeError '[] (+ "A" "Z")))""") + (raise (hy.errors.HyTypeError (+ "A" "Z") (, None '[] None)))""") assert "AZ" in err diff --git a/tests/test_lex.py b/tests/test_lex.py index 411c246c1..cc0dbf298 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -1,12 +1,18 @@ # Copyright 2018 the authors. # This file is part of Hy, which is free software licensed under the Expat # license. See the LICENSE. +import sys +import traceback + +import pytest from math import isnan from hy.models import (HyExpression, HyInteger, HyFloat, HyComplex, HySymbol, HyString, HyDict, HyList, HySet, HyKeyword) -from hy.lex import LexException, PrematureEndOfInput, tokenize -import pytest +from hy.lex import tokenize +from hy.lex.exceptions import LexException, PrematureEndOfInput +from hy.errors import hy_exc_handler + def peoi(): return pytest.raises(PrematureEndOfInput) def lexe(): return pytest.raises(LexException) @@ -29,8 +35,18 @@ def test_unbalanced_exception(): def test_lex_single_quote_err(): "Ensure tokenizing \"' \" throws a LexException that can be stringified" # https://github.com/hylang/hy/issues/1252 - with lexe() as e: tokenize("' ") - assert "Could not identify the next token" in str(e.value) + with lexe() as execinfo: + tokenize("' ") + + expected = [' File "", line -1\n', + " '\n", + ' ^\n', + 'LexException: Could not identify the next token.\n'] + output = traceback.format_exception_only(execinfo.type, execinfo.value) + + assert output[:-1:1] == expected[:-1:1] + # Python 2.7 doesn't give the full exception name, so we compensate. + assert output[-1].endswith(expected[-1]) def test_lex_expression_symbols(): @@ -73,7 +89,16 @@ def test_lex_strings_exception(): """ Make sure tokenize throws when codec can't decode some bytes""" with lexe() as execinfo: tokenize('\"\\x8\"') - assert "Can't convert \"\\x8\" to a HyString" in str(execinfo.value) + + expected = [' File "", line 1\n', + ' "\\x8"\n', + ' ^\n', + 'LexException: Can\'t convert "\\x8" to a HyString\n'] + output = traceback.format_exception_only(execinfo.type, execinfo.value) + + assert output[:-1:1] == expected[:-1:1] + # Python 2.7 doesn't give the full exception name, so we compensate. + assert output[-1].endswith(expected[-1]) def test_lex_bracket_strings(): @@ -179,13 +204,30 @@ def test_lex_digit_separators(): def test_lex_bad_attrs(): - with lexe(): tokenize("1.foo") + with lexe() as execinfo: + tokenize("1.foo") + + expected = [ + ' File "", line 1\n', + ' 1.foo\n', + ' ^\n', + ('LexException: Cannot access attribute on anything other' + ' than a name (in order to get attributes of expressions,' + ' use `(. )` or `(. )`)\n') + ] + output = traceback.format_exception_only(execinfo.type, execinfo.value) + + assert output[:-1:1] == expected[:-1:1] + # Python 2.7 doesn't give the full exception name, so we compensate. + assert output[-1].endswith(expected[-1]) + with lexe(): tokenize("0.foo") with lexe(): tokenize("1.5.foo") with lexe(): tokenize("1e3.foo") with lexe(): tokenize("5j.foo") with lexe(): tokenize("3+5j.foo") with lexe(): tokenize("3.1+5.1j.foo") + assert tokenize("j.foo") with lexe(): tokenize("3/4.foo") assert tokenize("a/1.foo") @@ -418,3 +460,51 @@ def test_discard(): assert tokenize("a '#_b c") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("c")])] assert tokenize("a '#_b #_c d") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("d")])] assert tokenize("a '#_ #_b c d") == [HySymbol("a"), HyExpression([HySymbol("quote"), HySymbol("d")])] + + + +def test_lex_exception_filtering(capsys): + """Confirm that the exception filtering works for lexer errors.""" + + def _check_trace_output(execinfo, expected): + sys.__excepthook__(execinfo.type, execinfo.value, execinfo.tb) + captured_wo_filtering = capsys.readouterr()[-1].strip('\n') + + hy_exc_handler(execinfo.type, execinfo.value, execinfo.tb) + captured_w_filtering = capsys.readouterr()[-1].strip('\n') + + output = captured_w_filtering.split('\n') + + # Make sure the filtered frames aren't the same as the unfiltered ones. + assert output[:-1:1] != captured_wo_filtering.split('\n')[:-1:1] + # Remove the origin frame lines. + assert output[3:-1:1] == expected[:-1:1] + # Python 2.7 doesn't give the full exception name, so we compensate. + assert output[-1].endswith(expected[-1]) + + + # First, test for PrematureEndOfInput + with peoi() as execinfo: + tokenize(" \n (foo") + + expected = [' File "", line 2', + ' (foo', + ' ^', + 'PrematureEndOfInput: Premature end of input'] + + _check_trace_output(execinfo, expected) + + # Now, for a generic LexException + with lexe() as execinfo: + tokenize(" \n\n 1.foo ") + + expected = [ + ' File "", line 3', + ' 1.foo', + ' ^', + ('LexException: Cannot access attribute on anything other' + ' than a name (in order to get attributes of expressions,' + ' use `(. )` or `(. )`)') + ] + + _check_trace_output(execinfo, expected)