Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Avoid macro-expand recursion into Expr(:toplevel, ...) #53515

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ Language changes
may pave the way for inference to be able to intelligently re-use the old
results, once the new method is deleted. ([#53415])

- Macro expansion will no longer eargerly recurse into into `Expr(:toplevel)`
expressions returned from macros. Instead, macro expansion of `:toplevel`
expressions will be delayed until evaluation time. This allows a later
expression within a given `:toplevel` expression to make use of macros
defined earlier in the same `:toplevel` expression. ([#53515])

Compiler/Runtime improvements
-----------------------------

Expand Down
108 changes: 97 additions & 11 deletions base/docs/Docs.jl
Original file line number Diff line number Diff line change
Expand Up @@ -446,24 +446,76 @@ more than one expression is marked then the same docstring is applied to each ex
end

`@__doc__` has no effect when a macro that uses it is not documented.

!!! compat "Julia 1.12"

This section documents a very subtle corner case that is only relevant to
macros which themselves both define other macros and then attempt to use them
within the same expansion. Such macros were impossible to write prior to
Julia 1.12 and are still quite rare. If you are not writing such a macro,
you may ignore this note.

In versions prior to Julia 1.12, macroexpansion would recursively expand through
`Expr(:toplevel)` blocks. This behavior was changed in Julia 1.12 to allow
macros to recursively define other macros and use them in the same returned
expression. However, to preserve backwards compatibility with existing uses of
`@__doc__`, the doc system will still expand through `Expr(:toplevel)` blocks
when looking for `@__doc__` markers. As a result, macro-defining-macros will
have an observable behavior difference when annotated with a docstring:

```julia
julia> macro macroception()
Expr(:toplevel, :(macro foo() 1 end), :(@foo))
end

julia> @macroception
1

julia> "Docstring" @macroception
ERROR: LoadError: UndefVarError: `@foo` not defined in `Main`
```

The supported workaround is to manually expand the `@__doc__` macro in the
defining macro, which the docsystem will recognize and suppress the recursive
expansion:

```julia
julia> macro macroception()
Expr(:toplevel,
macroexpand(__module__, :(@__doc__ macro foo() 1 end); recursive=false),
:(@foo))
end

julia> @macroception
1

julia> "Docstring" @macroception
1
```
"""
:(Core.@__doc__)

function __doc__!(source, mod, meta, def, define::Bool)
@nospecialize source mod meta def
# Two cases must be handled here to avoid redefining all definitions contained in `def`:
if define
# `def` has not been defined yet (this is the common case, i.e. when not generating
# the Base image). We just need to convert each `@__doc__` marker to an `@doc`.
finddoc(def) do each
function replace_meta_doc(each)
each.head = :macrocall
each.args = Any[Symbol("@doc"), source, mod, nothing, meta, each.args[end], define]
end

# `def` has not been defined yet (this is the common case, i.e. when not generating
# the Base image). We just need to convert each `@__doc__` marker to an `@doc`.
found = finddoc(replace_meta_doc, mod, def; expand_toplevel = false)

if !found
found = finddoc(replace_meta_doc, mod, def; expand_toplevel = true)
end
else
# `def` has already been defined during Base image gen so we just need to find and
# document any subexpressions marked with `@__doc__`.
docs = []
found = finddoc(def) do each
found = finddoc(mod, def; expand_toplevel = true) do each
push!(docs, :(@doc($source, $mod, $meta, $(each.args[end]), $define)))
end
# If any subexpressions have been documented then replace the entire expression with
Expand All @@ -472,25 +524,30 @@ function __doc__!(source, mod, meta, def, define::Bool)
def.head = :toplevel
def.args = docs
end
found
end
return found
end
# Walk expression tree `def` and call `λ` when any `@__doc__` markers are found. Returns
# `true` to signify that at least one `@__doc__` has been found, and `false` otherwise.
function finddoc(λ, def::Expr)
function finddoc(λ, mod::Module, def::Expr; expand_toplevel::Bool=false)
if isexpr(def, :block, 2) && isexpr(def.args[1], :meta, 1) && (def.args[1]::Expr).args[1] === :doc
# Found the macroexpansion of an `@__doc__` expression.
λ(def)
true
else
if expand_toplevel && isexpr(def, :toplevel)
for i = 1:length(def.args)
def.args[i] = macroexpand(mod, def.args[i])
end
end
found = false
for each in def.args
found |= finddoc(λ, each)
found |= finddoc(λ, mod, each; expand_toplevel)
end
found
end
end
finddoc(λ, @nospecialize def) = false
finddoc(λ, mod::Module, @nospecialize def; expand_toplevel::Bool=false) = false

# Predicates and helpers for `docm` expression selection:

Expand Down Expand Up @@ -528,8 +585,37 @@ iscallexpr(ex) = false
function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true)
@nospecialize meta ex
# Some documented expressions may be decorated with macro calls which obscure the actual
# expression. Expand the macro calls and remove extra blocks.
x = unblock(macroexpand(mod, ex))
# expression. Expand the macro calls.
x = macroexpand(mod, ex)
return _docm(source, mod, meta, x, define)
end

function _docm(source::LineNumberNode, mod::Module, meta, x, define::Bool = true)
if isexpr(x, :var"hygienic-scope")
x.args[1] = _docm(source, mod, meta, x.args[1])
return x
elseif isexpr(x, :escape)
x.args[1] = _docm(source, mod, meta, x.args[1])
return x
elseif isexpr(x, :block)
docarg = 0
for i = 1:length(x.args)
isa(x.args[i], LineNumberNode) && continue
if docarg == 0
docarg = i
continue
end
# More than one documentable expression in the block, treat it as a whole
# expression, which will fall through and look for (Expr(:meta, doc))
docarg = 0
break
end
if docarg != 0
x.args[docarg] = _docm(source, mod, meta, x.args[docarg], define)
return x
end
end

# Don't try to redefine expressions. This is only needed for `Base` img gen since
# otherwise calling `loaddocs` would redefine all documented functions and types.
def = define ? x : nothing
Expand Down Expand Up @@ -594,7 +680,7 @@ function docm(source::LineNumberNode, mod::Module, meta, ex, define::Bool = true
# All other expressions are undocumentable and should be handled on a case-by-case basis
# with `@__doc__`. Unbound string literals are also undocumentable since they cannot be
# retrieved from the module's metadata `IdDict` without a reference to the string.
docerror(ex)
docerror(x)

return doc
end
Expand Down
2 changes: 1 addition & 1 deletion src/ast.c
Original file line number Diff line number Diff line change
Expand Up @@ -1154,7 +1154,7 @@ static jl_value_t *jl_expand_macros(jl_value_t *expr, jl_module_t *inmodule, str
jl_expr_t *e = (jl_expr_t*)expr;
if (e->head == jl_inert_sym ||
e->head == jl_module_sym ||
//e->head == jl_toplevel_sym || // TODO: enable this once julia-expand-macroscope is fixed / removed
e->head == jl_toplevel_sym ||
e->head == jl_meta_sym) {
return expr;
}
Expand Down
Loading
Loading