Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adjust to JuliaLang/julia#53219 #547

Merged
merged 6 commits into from
Mar 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Project.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "Cthulhu"
uuid = "f68482b8-f384-11e8-15f7-abe071a5a75f"
authors = ["Valentin Churavy <[email protected]> and contributors"]
version = "2.11.1"
version = "2.12.0"

[deps]
CodeTracking = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
Expand All @@ -24,7 +24,7 @@ JuliaSyntax = "0.4"
PrecompileTools = "1"
Preferences = "1"
REPL = "1.9"
TypedSyntax = "1.2.2"
TypedSyntax = "1.3.0"
UUIDs = "1.9"
Unicode = "1.9"
WidthLimitedIO = "1"
Expand Down
102 changes: 73 additions & 29 deletions TypedSyntax/src/node.jl
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,23 @@ const no_default_value = NoDefaultValue()
# These are TypedSyntaxNode constructor helpers
# Call these directly if you want both the TypedSyntaxNode and the `mappings` list,
# where `mappings[i]` corresponds to the list of nodes matching `(src::CodeInfo).code[i]`.
function tsn_and_mappings(@nospecialize(f), @nospecialize(t); kwargs...)
m = which(f, t)
src, rt = getsrc(f, t)
tsn_and_mappings(m, src, rt; kwargs...)
function tsn_and_mappings(@nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...)
inferred_result = get_inferred_result(f, tt)
return tsn_and_mappings(inferred_result.mi, inferred_result.src, inferred_result.rt; kwargs...)
end

function tsn_and_mappings(m::Method, src::CodeInfo, @nospecialize(rt); warn::Bool=true, strip_macros::Bool=false, kwargs...)
function tsn_and_mappings(mi::MethodInstance, src::CodeInfo, @nospecialize(rt); warn::Bool=true, strip_macros::Bool=false, kwargs...)
m = mi.def::Method
def = definition(String, m)
if isnothing(def)
warn && @warn "couldn't retrieve source of $m"
return nothing, nothing
end
return tsn_and_mappings(m, src, rt, def...; warn, strip_macros, kwargs...)
return tsn_and_mappings(mi, src, rt, def...; warn, strip_macros, kwargs...)
end

function tsn_and_mappings(m::Method, src::CodeInfo, @nospecialize(rt), sourcetext::AbstractString, lineno::Integer; warn::Bool=true, strip_macros::Bool=false, kwargs...)
function tsn_and_mappings(mi::MethodInstance, src::CodeInfo, @nospecialize(rt), sourcetext::AbstractString, lineno::Integer; warn::Bool=true, strip_macros::Bool=false, kwargs...)
m = mi.def::Method
filename = isnothing(functionloc(m)[1]) ? string(m.file) : functionloc(m)[1]
rootnode = JuliaSyntax.parsestmt(SyntaxNode, sourcetext; filename=filename, first_line=lineno, kwargs...)
if strip_macros
Expand All @@ -50,22 +51,26 @@ function tsn_and_mappings(m::Method, src::CodeInfo, @nospecialize(rt), sourcetex
end
end
Δline = lineno - m.line # offset from original line number (Revise)
mappings, symtyps = map_ssas_to_source(src, rootnode, Δline)
mappings, symtyps = map_ssas_to_source(src, mi, rootnode, Δline)
node = TypedSyntaxNode(rootnode, src, mappings, symtyps)
node.typ = rt
return node, mappings
end

TypedSyntaxNode(@nospecialize(f), @nospecialize(t); kwargs...) = tsn_and_mappings(f, t; kwargs...)[1]
TypedSyntaxNode(@nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...) = tsn_and_mappings(f, tt; kwargs...)[1]

function TypedSyntaxNode(mi::MethodInstance; kwargs...)
m = mi.def::Method
src, rt = getsrc(mi)
tsn_and_mappings(m, src, rt; kwargs...)[1]
src, rt = code_typed1_tsn(mi)
tsn_and_mappings(mi, src, rt; kwargs...)[1]
end

function TypedSyntaxNode(rootnode::SyntaxNode, @nospecialize(f), @nospecialize(tt=Base.default_tt(f)); kwargs...)
inferred_result = get_inferred_result(f, tt)
TypedSyntaxNode(rootnode, inferred_result.src, inferred_result.mi; kwargs...)
end

TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, Δline::Integer=0) =
TypedSyntaxNode(rootnode, src, map_ssas_to_source(src, rootnode, Δline)...)
TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, mi::MethodInstance, Δline::Integer=0) =
TypedSyntaxNode(rootnode, src, map_ssas_to_source(src, mi, rootnode, Δline)...)

function TypedSyntaxNode(rootnode::SyntaxNode, src::CodeInfo, mappings, symtyps)
# There may be ambiguous assignments back to the source; preserve just the unambiguous ones
Expand Down Expand Up @@ -304,17 +309,57 @@ function sparam_name(mi::MethodInstance, i::Int)
return sig.var.name
end

function getsrc(@nospecialize(f), @nospecialize(t))
srcrts = code_typed(f, t; debuginfo=:source, optimize=false)
return only(srcrts)
end

function getsrc(mi::MethodInstance)
cis = Base.code_typed_by_type(mi.specTypes; debuginfo=:source, optimize=false)
isempty(cis) && error("no applicable type-inferred code found for ", mi)
length(cis) == 1 || error("got $(length(cis)) possible type-inferred results for ", mi,
", you may need a more specialized signature")
return cis[1]::Pair{CodeInfo}
@static if isdefined(Base, :method_instances)
using Base: method_instances
else
function method_instances(@nospecialize(f), @nospecialize(t), world::UInt)
tt = Base.signature_type(f, t)
results = Core.MethodInstance[]
# this make a better error message than the typeassert that follows
world == typemax(UInt) && error("code reflection cannot be used from generated functions")
for match in Base._methods_by_ftype(tt, -1, world)::Vector
instance = Core.Compiler.specialize_method(match)
push!(results, instance)
end
return results
end
end

struct InferredResult
mi::MethodInstance
src::CodeInfo
rt
InferredResult(mi::MethodInstance, src::CodeInfo, @nospecialize(rt)) = new(mi, src, rt)
end
function get_inferred_result(@nospecialize(f), @nospecialize(tt=Base.default_tt(f)),
world::UInt=Base.get_world_counter())
mis = method_instances(f, tt, world)
if isempty(mis)
sig = sprint(Base.show_tuple_as_call, Symbol(""), Base.signature_type(f, tt))
error("no applicable type-inferred code found for ", sig)
elseif length(mis) ≠ 1
sig = sprint(Base.show_tuple_as_call, Symbol(""), Base.signature_type(f, tt))
error("got $(length(mis)) possible type-inferred results for ", sig,
", you may need a more specialized signature")
end
mi = only(mis)
return InferredResult(mi, code_typed1_tsn(mi)...)
end

code_typed1_tsn(mi::MethodInstance) = code_typed1_by_method_instance(mi; optimize=false, debuginfo=:source)

function code_typed1_by_method_instance(mi::MethodInstance;
optimize::Bool=true,
debuginfo::Symbol=:default,
world::UInt=Base.get_world_counter(),
interp::Core.Compiler.AbstractInterpreter=Core.Compiler.NativeInterpreter(world))
(ccall(:jl_is_in_pure_context, Bool, ()) || world == typemax(UInt)) &&
error("code reflection should not be used from generated functions")
debuginfo = Base.IRShow.debuginfo(debuginfo)
code, rt = Core.Compiler.typeinf_code(interp, mi.def::Method, mi.specTypes, mi.sparam_vals, optimize)
code isa CodeInfo || error("no code is available for ", mi)
debuginfo === :none && Base.remove_linenums!(code)
return Pair{CodeInfo,Any}(code, rt)
end

function is_function_def(node) # this is not `Base.is_function_def`
Expand Down Expand Up @@ -397,8 +442,7 @@ end
# Main logic for mapping `src.code[i]` to node(s) in the SyntaxNode tree
# Success: when we map it to a unique node
# Δline is the (Revise) offset of the line number
function map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Int)
mi = src.parent::MethodInstance
function map_ssas_to_source(src::CodeInfo, mi::MethodInstance, rootnode::SyntaxNode, Δline::Int)
slottypes = src.slottypes::Union{Nothing, Vector{Any}}
have_slottypes = slottypes !== nothing
ssavaluetypes = src.ssavaluetypes::Vector{Any}
Expand Down Expand Up @@ -428,7 +472,7 @@ function map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Int)
# (Essentially `copy!(mapped, filter(predicate, targets))`)
function append_targets_for_line!(mapped#=::Vector{nodes}=#, i::Int, targets#=::Vector{nodes}=#)
j = src.codelocs[i]
lt = src.linetable::Vector{Any}
lt = src.linetable::Vector
start = getline(lt, j) + Δline
stop = getnextline(lt, j, Δline) - 1
linerange = start : stop
Expand Down Expand Up @@ -736,7 +780,7 @@ function map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Int)
end
return mappings, symtyps
end
map_ssas_to_source(src::CodeInfo, rootnode::SyntaxNode, Δline::Integer) = map_ssas_to_source(src, rootnode, Int(Δline))
map_ssas_to_source(src::CodeInfo, mi::MethodInstance, rootnode::SyntaxNode, Δline::Integer) = map_ssas_to_source(src, mi, rootnode, Int(Δline))

function follow_back(src, arg)
# Follow SSAValue backward to see if it maps back to a slot
Expand Down
2 changes: 1 addition & 1 deletion TypedSyntax/test/exhaustive.jl
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ const goodmis = Core.MethodInstance[]
continue
end
try
tsn, _ = TypedSyntax.tsn_and_mappings(m, src, rt, ret...; warn=false)
tsn, _ = TypedSyntax.tsn_and_mappings(mi, src, rt, ret...; warn=false)
@test isa(tsn, TypedSyntaxNode)
push!(goodmis, mi)
catch
Expand Down
40 changes: 16 additions & 24 deletions TypedSyntax/test/runtests.jl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using JuliaSyntax: JuliaSyntax, SyntaxNode, children, child, sourcetext, kind, @K_str
using TypedSyntax: TypedSyntax, TypedSyntaxNode, getsrc
using TypedSyntax: TypedSyntax, TypedSyntaxNode
using Dates, InteractiveUtils, Test

has_name_typ(node, name::Symbol, @nospecialize(T)) = kind(node) == K"Identifier" && node.val === name && node.typ === T
Expand All @@ -15,8 +15,7 @@ include("test_module.jl")
"""
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN1.jl")
TSN.eval(Expr(rootnode))
src, _ = getsrc(TSN.f, (Float32, Int, Float64))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, TSN.f, (Float32, Int, Float64))
sig, body = children(tsn)
@test children(sig)[2].typ === Float32
@test children(sig)[3].typ === Int
Expand All @@ -33,8 +32,7 @@ include("test_module.jl")
"""
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN2.jl")
TSN.eval(Expr(rootnode))
src, _ = getsrc(TSN.g, (Int16, Int16, Int32))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, TSN.g, (Int16, Int16, Int32))
sig, body = children(tsn)
@test length(children(sig)) == 4
@test children(body)[2].typ === Int32
Expand All @@ -46,8 +44,7 @@ include("test_module.jl")
st = "math(x) = x + sin(x + π / 4)"
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN2.jl")
TSN.eval(Expr(rootnode))
src, _ = getsrc(TSN.math, (Int,))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, TSN.math, (Int,))
sig, body = children(tsn)
@test has_name_typ(child(body, 1), :x, Int)
@test has_name_typ(child(body, 3, 2, 1), :x, Int)
Expand All @@ -70,8 +67,7 @@ include("test_module.jl")
st = "math2(x) = sin(x) + sin(x)"
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN2.jl")
TSN.eval(Expr(rootnode))
src, _ = getsrc(TSN.math2, (Int,))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, TSN.math2, (Int,))
sig, body = children(tsn)
@test body.typ === Float64
@test_broken child(body, 1).typ === Float64
Expand All @@ -91,8 +87,7 @@ include("test_module.jl")
)
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN3.jl")
TSN.eval(Expr(rootnode))
src, _ = getsrc(TSN.firstfirst, (Vector{Vector{Real}},))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, TSN.firstfirst, (Vector{Vector{Real}},))
sig, body = children(tsn)
@test child(body, idxsinner...).typ === nothing
@test child(body, idxsouter...).typ === Vector{Real}
Expand Down Expand Up @@ -150,8 +145,7 @@ include("test_module.jl")
"""
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN4.jl")
TSN.eval(Expr(rootnode))
src, rt = getsrc(TSN.setlist!, (Vector{Vector{Float32}}, Vector{Vector{UInt8}}, Int, Int))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, TSN.setlist!, (Vector{Vector{Float32}}, Vector{Vector{UInt8}}, Int, Int))
sig, body = children(tsn)
nodelist = child(body, 1, 2, 1, 1) # `listget`
@test sourcetext(nodelist) == "listget" && nodelist.typ === Vector{Vector{UInt8}}
Expand All @@ -175,8 +169,7 @@ include("test_module.jl")
"""
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN5.jl")
TSN.eval(Expr(rootnode))
src, rt = getsrc(TSN.callfindmin, (Vector{Float64},))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, TSN.callfindmin, (Vector{Float64},))
sig, body = children(tsn)
t = child(body, 1, 1)
@test kind(t) == K"tuple"
Expand Down Expand Up @@ -280,18 +273,18 @@ include("test_module.jl")
"""
rootnode = JuliaSyntax.parsestmt(SyntaxNode, st; filename="TSN6.jl")
TSN.eval(Expr(rootnode))
src, rt = getsrc(TSN.avoidzero, (Int,))
inferred_result = TypedSyntax.get_inferred_result(TSN.avoidzero, (Int,))
src, rt, mi = inferred_result.src, inferred_result.rt, inferred_result.mi
# src looks like this:
# %1 = Main.TSN.:(var"#avoidzero#6")(true, #self#, x)::Float64
# return %1
# Consequently there is nothing to match, but at least we shouldn't error
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, src, mi)
@test isa(tsn, TypedSyntaxNode)
@test rt === Float64
# Try the kwbodyfunc
m = which(TSN.avoidzero, (Int,))
src, rt = getsrc(Base.bodyfunction(m), (Bool, typeof(TSN.avoidzero), Int,))
tsn = TypedSyntaxNode(rootnode, src)
tsn = TypedSyntaxNode(rootnode, Base.bodyfunction(m), (Bool, typeof(TSN.avoidzero), Int,))
sig, body = children(tsn)
isz = child(body, 2, 1, 1)
@test kind(isz) == K"call" && child(isz, 1).val == :iszero
Expand Down Expand Up @@ -520,8 +513,7 @@ include("test_module.jl")
@test_broken body.typ == Int

# Construction from MethodInstance
src, rt = TypedSyntax.getsrc(TSN.myoftype, (Float64, Int))
tsn = TypedSyntaxNode(src.parent)
tsn = TypedSyntaxNode(TSN.myoftype, (Float64, Int))
sig, body = children(tsn)
node = child(body, 1)
@test node.typ === Type{Float64}
Expand Down Expand Up @@ -641,10 +633,10 @@ include("test_module.jl")
@test isa(tsnc, TypedSyntaxNode)

# issue 487
m = which(TSN.f487, (Int,))
src, rt = getsrc(TSN.f487, (Int,))
inferred_result = TypedSyntax.get_inferred_result(TSN.f487, (Int,))
src, mi = inferred_result.src, inferred_result.mi
rt = Core.Const(1)
tsn, _ = TypedSyntax.tsn_and_mappings(m, src, rt)
tsn, _ = TypedSyntax.tsn_and_mappings(mi, src, rt)
@test_nowarn str = sprint(tsn; context=:color=>false) do io, obj
printstyled(io, obj; hide_type_stable=false)
end
Expand Down
1 change: 0 additions & 1 deletion src/codeview.jl
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,6 @@ function cthulhu_typed(io::IO, debuginfo::Symbol,
# we're working on pre-optimization state, need to ignore `LimitedAccuracy`
src = copy(src)
src.ssavaluetypes = mapany(ignorelimited, src.ssavaluetypes::Vector{Any})
src.rettype = ignorelimited(src.rettype)

if src.slotnames !== nothing
slotnames = Base.sourceinfo_slotnames(src)
Expand Down
29 changes: 27 additions & 2 deletions src/interpreter.jl
Original file line number Diff line number Diff line change
Expand Up @@ -131,12 +131,37 @@ function create_cthulhu_source(@nospecialize(opt), effects::Effects)
return OptimizedSource(ir, opt.src, opt.src.inlineable, effects)
end

@static if VERSION ≥ v"1.12.0-DEV.15"
function CC.transform_result_for_cache(interp::CthulhuInterpreter,
linfo::MethodInstance, valid_worlds::WorldRange, result::InferenceResult, can_discard_trees::Bool=false)
return create_cthulhu_source(result.src, result.ipo_effects)
end
else
function CC.transform_result_for_cache(interp::CthulhuInterpreter,
linfo::MethodInstance, valid_worlds::WorldRange, result::InferenceResult)
return create_cthulhu_source(result.src, result.ipo_effects)
end
end

@static if VERSION ≥ v"1.11.0-DEV.879"
@static if VERSION ≥ v"1.12.0-DEV.45"
function CC.src_inlining_policy(interp::CthulhuInterpreter,
@nospecialize(src), @nospecialize(info::CCCallInfo), stmt_flag::UInt32)
if isa(src, OptimizedSource)
if CC.is_stmt_inline(stmt_flag) || src.isinlineable
return true
end
return false
else
@assert src isa CC.IRCode || src === nothing "invalid Cthulhu code cache"
# the default inlining policy may try additional effor to find the source in a local cache
return @invoke CC.src_inlining_policy(interp::AbstractInterpreter,
src::Any, info::CCCallInfo, stmt_flag::UInt32)
end
end
CC.retrieve_ir_for_inlining(cached_result::CodeInstance, src::OptimizedSource) = CC.copy(src.ir)
CC.retrieve_ir_for_inlining(mi::Core.MethodInstance, src::OptimizedSource, preserve_local_sources::Bool) =
CC.retrieve_ir_for_inlining(mi, src.ir, preserve_local_sources)
elseif VERSION ≥ v"1.11.0-DEV.879"
function CC.inlining_policy(interp::CthulhuInterpreter,
@nospecialize(src), @nospecialize(info::CCCallInfo), stmt_flag::UInt32)
if isa(src, OptimizedSource)
Expand Down Expand Up @@ -181,7 +206,7 @@ function CC.IRInterpretationState(interp::CthulhuInterpreter,
src = inferred.src
method_info = CC.MethodInfo(src)
return CC.IRInterpretationState(interp, method_info, ir, mi, argtypes, world,
src.min_world, src.max_world)
code.min_world, code.max_world)
end

@static if VERSION ≥ v"1.11.0-DEV.737"
Expand Down
7 changes: 3 additions & 4 deletions src/reflection.jl
Original file line number Diff line number Diff line change
Expand Up @@ -350,13 +350,12 @@ function add_sourceline!(locs, CI, stmtidx::Int)
end

function get_typed_sourcetext(mi::MethodInstance, src::CodeInfo, @nospecialize(rt); warn::Bool=true)
meth = mi.def::Method
tsn, mappings = TypedSyntax.tsn_and_mappings(meth, src, rt; warn, strip_macros=true)
return truncate_if_defaultargs!(tsn, mappings, meth)
tsn, mappings = TypedSyntax.tsn_and_mappings(mi, src, rt; warn, strip_macros=true)
return truncate_if_defaultargs!(tsn, mappings, mi.def::Method)
end

function get_typed_sourcetext(mi::MethodInstance, ::IRCode, @nospecialize(rt); kwargs...)
src, rt = TypedSyntax.getsrc(mi)
src, rt = TypedSyntax.code_typed1_tsn(mi)
return get_typed_sourcetext(mi, src, rt; kwargs...)
end

Expand Down
Loading
Loading