Merge pull request #2494 from scauligi/reread

Always have a reasonable reader accessible

NEWS.rst

@@ -16,6 +16,8 @@ Breaking Changes
 New Features
 ------------------------------
 * `defn`, `defn/a`, and `defclass` now support type parameters.
+* `HyReader` now has an optional parameter to install existing
+  reader macros from the calling module.
 
 Misc. Improvements
 ------------------------------
@@ -28,6 +30,8 @@ Bug Fixes
 * Fixed incomplete recognition of macro calls with a unary dotted
   head like `((. defn) f [])`.
 * `~@ #*` now produces a syntax error instead of a nonsensical result.
+* Fixed `hy.eval` failing on `defreader` or `require` forms that
+  install a new reader.
 
 0.27.0 (released 2023-07-06)
 =============================

hy/cmdline.py

@@ -372,16 +372,18 @@ def hy2py_worker(source, options, filename=None, parent_module=None, output_file
     ) as output_file:
 
         def printing_source(hst):
-            for node in hst:
-                if options.with_source:
-                    print(node, file=output_file)
-                yield node
-
-        hst = hy.models.Lazy(
-            printing_source(read_many(source, filename, skip_shebang=True))
-        )
-        hst.source = source
-        hst.filename = filename
+            def _printing_gen(hst):
+                for node in hst:
+                    if options.with_source:
+                        print(node, file=output_file)
+                    yield node
+            printing_hst = hy.models.Lazy(_printing_gen(hst))
+            printing_hst.source = hst.source
+            printing_hst.filename = hst.filename
+            printing_hst.reader = hst.reader
+            return printing_hst
+
+        hst = printing_source(read_many(source, filename, skip_shebang=True))
 
         with filtered_hy_exceptions():
             module_name = source_path.stem if source_path else Path(filename).name

hy/compiler.py

@@ -31,7 +31,7 @@
     as_model,
     is_unpack,
 )
-from hy.reader import mangle
+from hy.reader import mangle, HyReader
 from hy.scoping import ScopeGlobal
 
 hy_ast_compile_flags = 0
@@ -795,6 +795,7 @@ def hy_compile(
 
     filename = getattr(tree, "filename", filename)
     source = getattr(tree, "source", source)
+    reader = getattr(tree, "reader", None)
 
     tree = as_model(tree)
     if not isinstance(tree, Object):
@@ -804,7 +805,7 @@ def hy_compile(
 
     compiler = compiler or HyASTCompiler(module, filename=filename, source=source)
 
-    with compiler.scope:
+    with HyReader.using_reader(reader, create=False), compiler.scope:
         result = compiler.compile(tree)
     expr = result.force_expr
 

hy/core/macros.hy

@@ -110,7 +110,7 @@
              ~@(if docstr [docstr] [])
              ~@body)))
        (eval-when-compile
-         (setv (get hy.&reader.reader-macros ~dispatch-key)
+         (setv (get (. (hy.reader.HyReader.current-reader) reader-macros) ~dispatch-key)
                (get _hy_reader_macros ~dispatch-key)))))
 
 

hy/core/result_macros.py

@@ -1845,7 +1845,7 @@ def compile_require(compiler, expr, root, entries):
                             mkexpr(
                                 dotted("hy.macros.enable-readers"),
                                 "None",
-                                dotted("hy.&reader"),
+                                mkexpr(dotted("hy.reader.HyReader.current-reader")),
                                 [reader_assignments],
                             ),
                         ),

hy/importer.py

@@ -11,7 +11,7 @@
 
 import hy
 from hy.compiler import hy_compile
-from hy.reader import read_many
+from hy.reader import read_many, HyReader
 
 
 @contextmanager
@@ -118,7 +118,7 @@ def _hy_source_to_code(self, data, path, _optimize=-1):
         if os.environ.get("HY_MESSAGE_WHEN_COMPILING"):
             print("Compiling", path, file=sys.stderr)
         source = data.decode("utf-8")
-        hy_tree = read_many(source, filename=path, skip_shebang=True)
+        hy_tree = read_many(source, filename=path, skip_shebang=True, reader=HyReader())
         with loader_module_obj(self) as module:
             data = hy_compile(hy_tree, module)
 
@@ -139,7 +139,7 @@ def _hy_compile_source(pathname, source):
         sys.modules[mname] = types.ModuleType(mname)
         return compile(
             hy_compile(
-                read_many(source.decode("UTF-8"), filename=pathname, skip_shebang=True),
+                read_many(source.decode("UTF-8"), filename=pathname, skip_shebang=True, reader=HyReader()),
                 sys.modules[mname],
             ),
             pathname,

hy/macros.py

@@ -32,7 +32,7 @@ def macro(name):
 
 def reader_macro(name, fn):
     fn = rename_function(fn, name)
-    inspect.getmodule(fn).__dict__.setdefault("_hy_reader_macros", {})[name] = fn
+    fn.__globals__.setdefault("_hy_reader_macros", {})[name] = fn
 
 
 def pattern_macro(names, pattern, shadow=None):
@@ -84,13 +84,12 @@ def wrapper(hy_compiler, *args):
 def install_macro(name, fn, module_of):
     name = mangle(name)
     fn = rename_function(fn, name)
-    calling_module = inspect.getmodule(module_of)
-    macros_obj = calling_module.__dict__.setdefault("_hy_macros", {})
+    macros_obj = module_of.__globals__.setdefault("_hy_macros", {})
     if name in getattr(builtins, "_hy_macros", {}):
         warnings.warn(
             (
                 f"{name} already refers to: `{name}` in module: `builtins`,"
-                f" being replaced by: `{calling_module.__name__}.{name}`"
+                f" being replaced by: `{module_of.__globals__.get('__name__', '(globals)')}.{name}`"
             ),
             RuntimeWarning,
             stacklevel=3,

hy/reader/__init__.py

@@ -33,10 +33,12 @@ def read_many(stream, filename="<string>", reader=None, skip_shebang=False):
     source = stream.read()
     stream.seek(pos)
 
-    m = hy.models.Lazy((reader or HyReader()).parse(
+    reader = reader or HyReader()
+    m = hy.models.Lazy(reader.parse(
         stream, filename, skip_shebang))
     m.source = source
     m.filename = filename
+    m.reader = reader
     return m
 
 
@@ -52,5 +54,5 @@ def read(stream, filename=None, reader=None):
     except StopIteration:
         raise EOFError()
     else:
-        m.source, m.filename = it.source, it.filename
+        m.source, m.filename, m.reader = it.source, it.filename, it.reader
         return m

hy/reader/hy_reader.py

@@ -1,6 +1,8 @@
 "Character reader for parsing Hy source."
 
 import codecs
+import inspect
+from contextlib import contextmanager, nullcontext
 from itertools import islice
 
 import hy
@@ -109,15 +111,19 @@ def err(msg):
 
 
 class HyReader(Reader):
-    """A modular reader for Hy source."""
+    """A modular reader for Hy source.
+
+    When ``use_current_readers`` is true, initialize this reader
+    with all reader macros from the calling module."""
 
     ###
     # Components necessary for Reader implementation
     ###
 
     NON_IDENT = set("()[]{};\"'`~")
+    _current_reader = None
 
-    def __init__(self):
+    def __init__(self, *, use_current_readers=False):
         super().__init__()
 
         # move any reader macros declared using
@@ -127,6 +133,31 @@ def __init__(self):
             if tag[0] == '#' and tag[1:]:
                 self.reader_macros[tag[1:]] = self.reader_table.pop(tag)
 
+        if use_current_readers:
+            self.reader_macros.update(
+                inspect.stack()[1].frame.f_globals.get("_hy_reader_macros", {})
+            )
+
+    @classmethod
+    def current_reader(cls, override=None, create=True):
+        return override or HyReader._current_reader or (cls() if create else None)
+
+    @contextmanager
+    def as_current_reader(self):
+        old_reader = HyReader._current_reader
+        HyReader._current_reader = self
+        try:
+            yield
+        finally:
+            HyReader._current_reader = old_reader
+
+    @classmethod
+    @contextmanager
+    def using_reader(cls, override=None, create=True):
+        reader = cls.current_reader(override, create)
+        with reader.as_current_reader() if reader else nullcontext():
+            yield
+
 
     def fill_pos(self, model, start):
         """Attach line/col information to a model.
@@ -159,8 +190,6 @@ def read_default(self, key):
     def parse(self, stream, filename=None, skip_shebang=False):
         """Yields all `hy.models.Object`'s in `source`
 
-        Additionally exposes `self` as ``hy.&reader`` during read/compile time.
-
         Args:
             source:
                 Hy source to be parsed.
@@ -178,17 +207,7 @@ def parse(self, stream, filename=None, skip_shebang=False):
                 if c == "\n":
                     break
 
-        rname = mangle("&reader")
-        old_reader = getattr(hy, rname, None)
-        setattr(hy, rname, self)
-
-        try:
-            yield from self.parse_forms_until("")
-        finally:
-            if old_reader is None:
-                delattr(hy, rname)
-            else:
-                setattr(hy, rname, old_reader)
+        yield from self.parse_forms_until("")
 
     ###
     # Reading forms
@@ -210,23 +229,28 @@ def try_parse_one_form(self):
                 fully parsing a form.
             LexException: If there is an error during form parsing.
         """
-        try:
-            self.slurp_space()
-            c = self.getc()
-            start = self._pos
-            if not c:
-                raise PrematureEndOfInput.from_reader(
-                    "Premature end of input while attempting to parse one form", self
+        with self.as_current_reader():
+            try:
+                self.slurp_space()
+                c = self.getc()
+                start = self._pos
+                if not c:
+                    raise PrematureEndOfInput.from_reader(
+                        "Premature end of input while attempting to parse one form", self
+                    )
+                handler = self.reader_table.get(c)
+                model = handler(self, c) if handler else self.read_default(c)
+                if model is not None:
+                    model = self.fill_pos(model, start)
+                    model.reader = self
+                    return model
+                return None
+            except LexException:
+                raise
+            except Exception as e:
+                raise LexException.from_reader(
+                    str(e) or "Exception thrown attempting to parse one form", self
                 )
-            handler = self.reader_table.get(c)
-            model = handler(self, c) if handler else self.read_default(c)
-            return self.fill_pos(model, start) if model is not None else None
-        except LexException:
-            raise
-        except Exception as e:
-            raise LexException.from_reader(
-                str(e) or "Exception thrown attempting to parse one form", self
-            )
 
     def parse_one_form(self):
         """Read from the stream until a form is parsed.

tests/native_tests/reader_macros.hy

@@ -4,6 +4,7 @@
   types
   contextlib [contextmanager]
   hy.errors [HyMacroExpansionError]
+  hy.reader [HyReader]
   hy.reader.exceptions [PrematureEndOfInput]
 
   pytest)
@@ -89,3 +90,45 @@
     (eval-module #[[(require tests.resources.tlib :readers [taggart] :readers [upper!])]]))
   (with [(pytest.raises hy.errors.HyRequireError)]
     (eval-module #[[(require tests.resources.tlib :readers [not-a-real-reader])]])))
+
+(defn test-eval-read []
+  ;; https://github.com/hylang/hy/issues/2291
+  ;; hy.eval should not raise an exception when
+  ;; defining readers using hy.read or with quoted forms
+  (with [module (temp-module "<test>")]
+    (hy.eval (hy.read "(defreader r 5)") :module module)
+    (hy.eval '(defreader test-read 4) :module module)
+    (hy.eval '(require tests.resources.tlib :readers [upper!]) :module module)
+    ;; these reader macros should not exist in any current reader
+    (for [tag #("#r" "#test-read" "#upper!")]
+      (with [(pytest.raises hy.errors.HySyntaxError)]
+        (hy.read tag)))
+    ;; but they should be installed in the module
+    (hy.eval '(setv reader (hy.reader.HyReader :use-current-readers True)) :module module)
+    (setv reader module.reader)
+    (for [[s val] [["#r" 5]
+                   ["#test-read" 4]
+                   ["#upper! \"hi there\"" "HI THERE"]]]
+      (assert (= (hy.eval (hy.read s :reader reader) :module module) val))))
+
+  ;; passing a reader explicitly should work as expected
+  (with [module (temp-module "<test>")]
+    (setv reader (HyReader))
+    (defn eval1 [s]
+      (hy.eval (hy.read s :reader reader) :module module))
+    (eval1 "(defreader fbaz 32)")
+    (eval1 "(require tests.resources.tlib :readers [upper!])")
+    (assert (= (eval1 "#fbaz") 32))
+    (assert (= (eval1 "#upper! \"hello\"") "HELLO"))))
+
+
+(defn test-interleaving-readers []
+  (with [module1 (temp-module "<one>")
+         module2 (temp-module "<two>")]
+    (setv stream1 (hy.read-many #[[(do (defreader foo "foo1") (defreader bar "bar1")) #foo #bar]])
+          stream2 (hy.read-many #[[(do (defreader foo "foo2") (defreader bar "bar2")) #foo #bar]])
+          valss [[None None] ["foo1" "foo2"] ["bar1" "bar2"]])
+    (for [[form1 form2 vals] (zip stream1 stream2 valss)]
+      (assert (= vals
+                 [(hy.eval form1 :module module1)
+                  (hy.eval form2 :module module2)])))))