diff --git a/.gitignore b/.gitignore index 71f8c81..b0019d5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ *.jl.mem expected.out failed.out +src/builtins.jl +deps/build.log +docs/build/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 0c440a6..0576f1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,5 +9,17 @@ julia: - 1.1 - nightly +script: + - julia --project -e 'using Pkg; + Pkg.instantiate(); + Pkg.add([PackageSpec(name = "TerminalRegressionTests", rev = "master"), + PackageSpec(name = "VT100", rev = "master")]); + Pkg.build(); + Pkg.test()' + +matrix: + allow_failures: + - julia: nightly + notifications: email: false diff --git a/Manifest.toml b/Manifest.toml index e9205e5..3d2d6bb 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -7,21 +7,14 @@ uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" deps = ["REPL"] git-tree-sha1 = "e4a4693fc3fd3924d469bb0fa46215672fd6d4b8" repo-rev = "master" -repo-url = "https://github.com/Keno/DebuggerFramework.jl.git" +repo-url = "https://github.com/JuliaDebug/DebuggerFramework.jl.git" uuid = "67417a49-6d77-5db2-98c7-c13144130cd2" version = "0.1.2+" -[[Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" @@ -30,16 +23,5 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" deps = ["InteractiveUtils", "Markdown", "Sockets"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/Project.toml b/Project.toml index 93c9e0d..ea11693 100644 --- a/Project.toml +++ b/Project.toml @@ -4,11 +4,14 @@ version = "0.1.1" [deps] DebuggerFramework = "67417a49-6d77-5db2-98c7-c13144130cd2" +InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [extras] +TerminalRegressionTests = "98bfdc55-cc95-5876-a49a-74609291cbe0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +VT100 = "7774df62-37c0-5c21-b34d-f6d7f98f54bc" [targets] -test = ["Test"] +test = ["Test", "TerminalRegressionTests", "VT100"] diff --git a/deps/build.jl b/deps/build.jl new file mode 100644 index 0000000..1f8979a --- /dev/null +++ b/deps/build.jl @@ -0,0 +1,7 @@ +using InteractiveUtils + +const srcpath = joinpath(dirname(@__DIR__), "src") +include(joinpath(srcpath, "generate_builtins.jl")) +open(joinpath(srcpath, "builtins.jl"), "w") do io + generate_builtins(io) +end diff --git a/docs/make.jl b/docs/make.jl new file mode 100644 index 0000000..4ec4dfa --- /dev/null +++ b/docs/make.jl @@ -0,0 +1,20 @@ +using Documenter, ASTInterpreter2 + +makedocs( + modules = [ASTInterpreter2], + clean = false, + format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"), + sitename = "ASTInterpreter2.jl", + authors = "Keno Fischer, Tim Holy, and others", + linkcheck = !("skiplinks" in ARGS), + pages = [ + "Home" => "index.md", + "ast.md", + "internals.md", + "dev_reference.md", + ], +) + +deploydocs( + repo = "github.com/JuliaDebug/ASTInterpreter2.jl.git", +) diff --git a/docs/src/ast.md b/docs/src/ast.md new file mode 100644 index 0000000..3259035 --- /dev/null +++ b/docs/src/ast.md @@ -0,0 +1,93 @@ +# Lowered representation + +Let's start with a demonstration on simple function: + +```julia +function summer(A::AbstractArray{T}) where T + s = zero(T) + for a in A + s += a + end + return s +end + +A = [1, 2, 5] +``` + +ASTIntepreter2 uses the lowered representation of code: + +```julia +julia> code = @code_lowered summer(A) +CodeInfo( +1 ─ s = (Main.zero)($(Expr(:static_parameter, 1))) +│ %2 = A +│ #temp# = (Base.iterate)(%2) +│ %4 = #temp# === nothing +│ %5 = (Base.not_int)(%4) +└── goto #4 if not %5 +2 ┄ %7 = #temp# +│ a = (Core.getfield)(%7, 1) +│ %9 = (Core.getfield)(%7, 2) +│ s = s + a +│ #temp# = (Base.iterate)(%2, %9) +│ %12 = #temp# === nothing +│ %13 = (Base.not_int)(%12) +└── goto #4 if not %13 +3 ─ goto #2 +4 ┄ return s +) +``` + +To understand this package's internals, you need to familiarize yourself with these +`CodeInfo` objects. The numbers on the left correspond to [basic blocks](https://en.wikipedia.org/wiki/Basic_block); +when used in statements these are printed with a hash, e.g., in `goto #4 if not %6`, the +`#4` refers to basic block 4. +The numbers in the next column--e.g., `%1`, refer to [single static assignment (SSA) values](https://en.wikipedia.org/wiki/Static_single_assignment_form). +Each statement (each line of this printout) corresponds to a single SSA value, +but only those used later in the code are printed using assignment syntax. +Wherever a previous SSA value is used, it's referenced by an `SSAValue` and printed as `%6`; +for example, in `goto #4 if not %6`, the `%6` is the result of evaluating the 6th statement, +which is `(Base.not_int)(%5)`, which in turn refers to the result of statement 5. +Together lines 5 and 6 correspond to `!(#temp# === nothing)`. +(The `#temp#` means that this was a generated variable name not present explicitly in the original source code.) + +Before diving into the details, let's first look at the statements themselves: + +```julia +julia> code.code +16-element Array{Any,1}: + :(_3 = (Main.zero)($(Expr(:static_parameter, 1)))) + :(_2) + :(_4 = (Base.iterate)(%2)) + :(_4 === nothing) + :((Base.not_int)(%4)) + :(unless %5 goto %16) + :(_4) + :(_5 = (Core.getfield)(%7, 1)) + :((Core.getfield)(%7, 2)) + :(_3 = _3 + _5) + :(_4 = (Base.iterate)(%2, %9)) + :(_4 === nothing) + :((Base.not_int)(%12)) + :(unless %13 goto %16) + :(goto %7) + :(return _3) +``` + +You can see directly that the SSA assignments are implicit; they are not directly +present in the statement list. +The most noteworthy change here is the appearance of objects like `_3`, which are +references that index into local variable slots: + +```julia +julia> code.slotnames +5-element Array{Any,1}: + Symbol("#self#") + :A + :s + Symbol("#temp#") + :a +``` + +When printing the whole `CodeInfo` object, these `slotnames` are substituted in. +The types of objects that can be in `code.code` is well-described in the [Julia AST](https://docs.julialang.org/en/latest/devdocs/ast/) documentation. diff --git a/docs/src/dev_reference.md b/docs/src/dev_reference.md new file mode 100644 index 0000000..24d7796 --- /dev/null +++ b/docs/src/dev_reference.md @@ -0,0 +1,55 @@ +# Function reference + +## Top-level + +```@docs +@interpret +``` + +## Frame creation + +```@docs +ASTInterpreter2.enter_call +ASTInterpreter2.enter_call_expr +ASTInterpreter2.build_frame +ASTInterpreter2.determine_method_for_expr +ASTInterpreter2.prepare_args +ASTInterpreter2.prepare_call +ASTInterpreter2.get_call_framecode +ASTInterpreter2.optimize! +``` + +## Frame execution + +```@docs +ASTInterpreter2.Compiled +ASTInterpreter2.step_expr! +ASTInterpreter2.finish! +ASTInterpreter2.finish_and_return! +ASTInterpreter2.next_until! +ASTInterpreter2.evaluate_call! +ASTInterpreter2.evaluate_foreigncall! +ASTInterpreter2.maybe_evaluate_builtin +ASTInterpreter2.@eval_rhs +``` + +## Types + +```@docs +ASTInterpreter2.JuliaStackFrame +ASTInterpreter2.JuliaFrameCode +ASTInterpreter2.JuliaProgramCounter +``` + +## Internal storage + +```@docs +ASTInterpreter2.framedict +ASTInterpreter2.genframedict +``` + +## Utilities + +```@docs +ASTInterpreter2.iswrappercall +``` diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..b24a6f1 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,26 @@ +# ASTInterpreter2 + +This package implements an [interpreter](https://en.wikipedia.org/wiki/Interpreter_(computing)) for Julia code. +Normally, Julia compiles your code when you first execute it; using ASTInterpreter2 you can +avoid compilation and execute the expressions that define your code directly. +Interpreters have a number of applications, including support for stepping debuggers. + +At a pure user level, there is not much to know: + +```jldoctest +julia> using ASTInterpreter2 + +julia> a = [1, 2, 5] +3-element Array{Int64,1}: + 1 + 2 + 5 + +julia> sum(a) +8 + +julia> @interpret sum(a) +8 +``` + +Those who want to dive deeper should continue reading. diff --git a/docs/src/internals.md b/docs/src/internals.md new file mode 100644 index 0000000..0fab6ed --- /dev/null +++ b/docs/src/internals.md @@ -0,0 +1,153 @@ +# Internals + +The process of executing code in the interpreter is to prepare a `frame` and then +evaluate these statements one-by-one, branching via the `goto` statements as appropriate. +Using the `summer` example described in [Lowered representation](@ref), +let's build a frame: + +```julia +julia> frame = ASTInterpreter2.enter_call(summer, A) +JuliaStackFrame(ASTInterpreter2.JuliaFrameCode(summer(A::AbstractArray{T,N} where N) where T in Main at REPL[1]:2, CodeInfo( +1 ─ s = ($(QuoteNode(zero)))($(Expr(:static_parameter, 1))) +│ %2 = A +│ #temp# = ($(QuoteNode(iterate)))(%2) +│ %4 = ($(QuoteNode(===)))(#temp#, nothing) +│ %5 = ($(QuoteNode(not_int)))(%4) +└── goto #4 if not %5 +2 ┄ %7 = #temp# +│ a = ($(QuoteNode(getfield)))(%7, 1) +│ %9 = ($(QuoteNode(getfield)))(%7, 2) +│ s = ($(QuoteNode(+)))(s, a) +│ #temp# = ($(QuoteNode(iterate)))(%2, %9) +│ %12 = ($(QuoteNode(===)))(#temp#, nothing) +│ %13 = ($(QuoteNode(not_int)))(%12) +└── goto #4 if not %13 +3 ─ goto #2 +4 ┄ return s +), Core.TypeMapEntry[#undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef], BitSet([2, 4, 5, 7, 9, 12, 13]), false, false, true), Union{Nothing, Some{Any}}[Some(summer), Some([1, 2, 5]), nothing, nothing, nothing], Any[#undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef, #undef], Any[Int64], Int64[], Base.RefValue{Any}(nothing), Base.RefValue{ASTInterpreter2.JuliaProgramCounter}(JuliaProgramCounter(1)), Dict{Symbol,Int64}(), Any[]) +``` + +This is a [`ASTInterpreter2.JuliaStackFrame`](@ref). The `CodeInfo` is the most prominent part of this display, +and extractable as `code = frame.code.code`. (It's a slightly modified form of one returned by `@code_lowered`, +in that it has been processed by [`ASTInterpreter2.optimize!`](@ref) to speed up run-time execution.) + +Much of the rest of the `frame` holds values needed for or generated by execution. +The input arguments are in `locals`: + +```julia +julia> frame.locals +5-element Array{Union{Nothing, Some{Any}},1}: + Some(summer) + Some([1, 2, 5]) + nothing + nothing + nothing +``` + +These correspond to the `code.slotnames`; the first is the `#self#` argument and the second +is the input array. The remaining local variables (e.g., `s` and `a`), have not yet been assigned---we've +only built the frame, but we haven't yet begun to execute it. +The static parameter, `T`, is stored in `frame.sparams`: + +```julia +julia> frame.sparams +1-element Array{Any,1}: + Int64 +``` + +The `Expr(:static_parameter, 1)` statement refers to this value. + +The other main storage is for the generated SSA values: + +```julia +julia> frame.ssavalues +16-element Array{Any,1}: + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef +``` + +Since we haven't executed any statements yet, these are all undefined. + +The other main entity is the so-called [program counter](https://en.wikipedia.org/wiki/Program_counter), +which just indicates the next statement to be executed: + +```julia +julia> frame.pc[] +JuliaProgramCounter(1) +``` + +This is stored as a `Ref` so that it can be updated as execution progresses. + +Let's try executing the first statement. So that we can recurse into calls (e.g., `iterate`, `+`, etc.,), +we'll create a [stack](https://en.wikipedia.org/wiki/Call_stack) of frames and then run the first statement: + +```julia +julia> stack = ASTInterpreter2.JuliaStackFrame[] +0-element Array{JuliaStackFrame,1} + +julia> ASTInterpreter2.step_expr!(stack, frame) +JuliaProgramCounter(2) +``` + +This indicates that it ran statement 1 and is prepared to run statement 2. +(It's worth noting that the first line included a `call` to `zero`, so behind the scenes +ASTInterpreter2 pushed this frame onto `stack`, created a new frame for `zero`, +executed all the statements, and then popped the stack.) +Since the first statement is an assignment of a local variable, let's check the +locals again: + +```julia +julia> frame.locals +5-element Array{Union{Nothing, Some{Any}},1}: + Some(summer) + Some([1, 2, 5]) + Some(0) + nothing + nothing +``` + +You can see that the entry corresponding to `s` has been initialized. + +The next statement just retrieves one of the slots (the input argument `A`) and stores +it in an SSA value: + +```julia +julia> ASTInterpreter2.step_expr!(stack, frame) +JuliaProgramCounter(3) + +julia> frame.ssavalues +16-element Array{Any,1}: + #undef + [1, 2, 5] + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef + #undef +``` + +One can easily continue this until execution completes, which is indicated when `step_expr!` +returns `nothing`. diff --git a/src/ASTInterpreter2.jl b/src/ASTInterpreter2.jl index f202eaa..7f6897a 100644 --- a/src/ASTInterpreter2.jl +++ b/src/ASTInterpreter2.jl @@ -1,4 +1,3 @@ -__precompile__() module ASTInterpreter2 using DebuggerFramework @@ -6,63 +5,173 @@ using DebuggerFramework: FileLocInfo, BufferLocInfo, Suppressed using Base.Meta using REPL.LineEdit using REPL -import Base: +, deepcopy_internal -using Core: CodeInfo, SSAValue, SlotNumber, TypeMapEntry, SimpleVector, LineInfoNode, GotoNode, Slot, GeneratedFunctionStub +import Base: +, convert, isless +using Core: CodeInfo, SSAValue, SlotNumber, TypeMapEntry, SimpleVector, LineInfoNode, GotoNode, Slot, + GeneratedFunctionStub, MethodInstance using Markdown -export @enter, @make_stack +export @enter, @make_stack, @interpret, Compiled, JuliaStackFrame -include("interpret.jl") +""" +`Compiled` is a trait indicating that any `:call` expressions should be evaluated +using Julia's normal compiled-code evaluation. The alternative is to pass `stack=JuliaStackFrame[]`, +which will cause all calls to be evaluated via the interpreter. +""" +struct Compiled end + +""" + JuliaProgramCounter(next_stmt::Int) +A wrapper specifying the index of the next statement in the lowered code to be executed. +""" struct JuliaProgramCounter next_stmt::Int end +(x::JuliaProgramCounter, y::Integer) = JuliaProgramCounter(x.next_stmt+y) +convert(::Type{Int}, pc::JuliaProgramCounter) = pc.next_stmt +isless(x::JuliaProgramCounter, y::Integer) = isless(x.next_stmt, y) -struct JuliaStackFrame - meth::Method +Base.show(io::IO, pc::JuliaProgramCounter) = print(io, "JuliaProgramCounter(", pc.next_stmt, ')') + +# A type used transiently in renumbering CodeInfo SSAValues (to distinguish a new SSAValue from an old one) +struct NewSSAValue + id::Int +end + +""" +`JuliaFrameCode` holds static information about a method or toplevel code. +One `JuliaFrameCode` can be shared by many `JuliaFrameState` calling frames. + +Important fields: +- `scope`: the `Method` or `Module` in which this frame is to be evaluated +- `code`: the `CodeInfo` object storing (optimized) lowered code +- `methodtables`: a vector, each entry potentially stores a "local method table" for the corresponding + `:call` expression in `code` (undefined entries correspond to statements that do not + contain `:call` expressions) +- `used`: a `BitSet` storing the list of SSAValues that get referenced by later statements. +""" +struct JuliaFrameCode + scope::Union{Method,Module} code::CodeInfo - locals::Vector{Any} - ssavalues::Vector{Any} + methodtables::Vector{TypeMapEntry} # line-by-line method tables for generic-function :call Exprs used::BitSet - sparams::Vector{Any} - exception_frames::Vector{Int} - last_exception::Ref{Any} - pc::JuliaProgramCounter - # A vector from names to the slotnumber of that name - # for which a reference was last encountered. - last_reference::Dict{Symbol, Int} wrapper::Bool generator::Bool # Display options fullpath::Bool end -function JuliaStackFrame(frame::JuliaStackFrame, pc::JuliaProgramCounter; wrapper = frame.wrapper, generator=frame.generator, fullpath=frame.fullpath) - JuliaStackFrame(frame.meth, frame.code, frame.locals, - frame.ssavalues, frame.used, frame.sparams, + +function JuliaFrameCode(frame::JuliaFrameCode; wrapper = frame.wrapper, generator=frame.generator, fullpath=frame.fullpath) + JuliaFrameCode(frame.scope, frame.code, frame.methodtables, frame.used, + wrapper, generator, fullpath) +end + +function JuliaFrameCode(scope, code::CodeInfo; wrapper=false, generator=false, fullpath=true) + code = optimize!(copy_codeinfo(code), moduleof(scope)) + used = find_used(code) + methodtables = Vector{TypeMapEntry}(undef, length(code.code)) + return JuliaFrameCode(scope, code, methodtables, used, wrapper, generator, fullpath) +end + +""" +`JuliaStackFrame` represents the current execution state in a particular call frame. + +Important fields: +- `code`: the [`JuliaFrameCode`](@ref) for this frame +- `locals`: a vector containing the input arguments and named local variables for this frame. + The indexing corresponds to the names in `frame.code.code.slotnames`. +- `ssavalues`: a vector containing the + [Static Single Assignment](https://en.wikipedia.org/wiki/Static_single_assignment_form) + values produced at the current state of execution +- `sparams`: the static type parameters, e.g., for `f(x::Vector{T}) where T` this would store + the value of `T` given the particular input `x`. +- `pc`: the [`JuliaProgramCounter`](@ref) that typically represents the current position + during execution. However, note that some internal functions instead maintain the `pc` + as a local variable, and only update the frame's `pc` when pushing a frame on the stack. +""" +struct JuliaStackFrame + code::JuliaFrameCode + locals::Vector{Union{Nothing,Some{Any}}} + ssavalues::Vector{Any} + sparams::Vector{Any} + exception_frames::Vector{Int} + last_exception::Base.RefValue{Any} + pc::Base.RefValue{JuliaProgramCounter} + # A vector from names to the slotnumber of that name + # for which a reference was last encountered. + last_reference::Dict{Symbol,Int} + callargs::Vector{Any} # a temporary for processing arguments of :call exprs +end + +function JuliaStackFrame(framecode::JuliaFrameCode, frame::JuliaStackFrame, pc::JuliaProgramCounter; kwargs...) + pcref = frame.pc + pcref[] = pc + if !isempty(kwargs) + framecode = JuliaFrameCode(framecode; kwargs...) + end + JuliaStackFrame(framecode, frame.locals, + frame.ssavalues, frame.sparams, frame.exception_frames, frame.last_exception, - pc, frame.last_reference, wrapper, generator, - fullpath) + pcref, frame.last_reference, frame.callargs) end +JuliaStackFrame(frame::JuliaStackFrame, pc::JuliaProgramCounter; kwargs...) = + JuliaStackFrame(frame.code, frame, pc; kwargs...) + +""" +`framedict[method]` returns the `JuliaFrameCode` for `method`. For `@generated` methods, +see [`genframedict`](@ref). +""" +const framedict = Dict{Method,JuliaFrameCode}() # essentially a method table for lowered code +""" +`genframedict[(method,argtypes)]` returns the `JuliaFrameCode` for a `@generated` method `method`, +for the particular argument types `argtypes`. + +The framecodes stored in `genframedict` are for the code returned by the generator +(i.e, what will run when you call the method on particular argument types); +for the generator itself, its framecode would be stored in [`framedict`](@ref). +""" +const genframedict = Dict{Tuple{Method,Type},JuliaFrameCode}() # the same for @generated functions + +const junk = JuliaStackFrame[] # to allow re-use of allocated memory (this is otherwise a bottleneck) + +include("localmethtable.jl") +include("interpret.jl") +include("builtins.jl") + +function moduleof(x) + if isa(x, JuliaStackFrame) + x = x.code.scope + end + return _moduleof(x) +end +_moduleof(scope::Method) = scope.module +_moduleof(scope::Module) = scope +Base.nameof(frame) = isa(frame.code.scope, Method) ? frame.code.scope.name : nameof(frame.code.scope) + is_loc_meta(expr, kind) = isexpr(expr, :meta) && length(expr.args) >= 1 && expr.args[1] === kind function DebuggerFramework.locdesc(frame::JuliaStackFrame, specslottypes = false) sprint() do io - argnames = frame.code.slotnames[2:frame.meth.nargs] - spectypes = Any[Any for i=1:length(argnames)] - print(io, frame.meth.name,'(') - first = true - for (argname, argT) in zip(argnames, spectypes) - first || print(io, ", ") - first = false - print(io, argname) - !(argT === Any) && print(io, "::", argT) + if frame.code.scope isa Method + meth = frame.code.scope + argnames = frame.code.code.slotnames[2:meth.nargs] + spectypes = Any[Any for i=1:length(argnames)] + print(io, meth.name,'(') + first = true + for (argname, argT) in zip(argnames, spectypes) + first || print(io, ", ") + first = false + print(io, argname) + !(argT === Any) && print(io, "::", argT) + end + print(io, ") at ", + frame.code.fullpath ? meth.file : + basename(String(meth.file)), + ":",meth.line) + else + println("not yet implemented") end - print(io, ") at ", - frame.fullpath ? frame.meth.file : - basename(String(frame.meth.file)), - ":",frame.meth.line) end end @@ -72,21 +181,27 @@ function DebuggerFramework.print_locals(io::IO, frame::JuliaStackFrame) # #self# is only interesting if it has values inside of it. We already know # which function we're in otherwise. val = something(frame.locals[i]) - if frame.code.slotnames[i] == Symbol("#self#") && (isa(val, Type) || sizeof(val) == 0) + if frame.code.code.slotnames[i] == Symbol("#self#") && (isa(val, Type) || sizeof(val) == 0) continue end - DebuggerFramework.print_var(io, frame.code.slotnames[i], something(frame.locals[i]), nothing) + DebuggerFramework.print_var(io, frame.code.code.slotnames[i], frame.locals[i], nothing) end end - for i = 1:length(frame.sparams) - DebuggerFramework.print_var(io, frame.meth.sparam_syms[i], frame.sparams[i], nothing) + if frame.code.scope isa Method + for i = 1:length(frame.sparams) + DebuggerFramework.print_var(io, frame.code.scope.sparam_syms[i], frame.sparams[i], nothing) + end end end const SEARCH_PATH = [] -__init__() = append!(SEARCH_PATH,[joinpath(Sys.BINDIR,"../share/julia/base/"), - joinpath(Sys.BINDIR,"../include/")]) +function __init__() + append!(SEARCH_PATH,[joinpath(Sys.BINDIR,"../share/julia/base/"), + joinpath(Sys.BINDIR,"../include/")]) + return nothing +end + function loc_for_fname(file, line, defline) if startswith(string(file),"REPL[") hist_idx = parse(Int,string(file)[6:end-1]) @@ -105,28 +220,30 @@ function loc_for_fname(file, line, defline) end function DebuggerFramework.locinfo(frame::JuliaStackFrame) - loc_for_fname(frame.meth.file, location(frame), frame.meth.line) -end - -function lookup_var_if_var(frame, x) - if isa(x, Union{SSAValue, GlobalRef, SlotNumber}) || isexpr(x, :static_parameter) || isexpr(x, :the_exception) || isexpr(x, :boundscheck) - return lookup_var(frame, x) + if frame.code.scope isa Method + meth = frame.code.scope + loc_for_fname(meth.file, location(frame), meth.line) + else + println("not yet implemented") end - x end function DebuggerFramework.eval_code(state, frame::JuliaStackFrame, command) expr = Base.parse_input_line(command) + if isexpr(expr, :toplevel) + expr = expr.args[end] + end local_vars = Any[] local_vals = Any[] for i = 1:length(frame.locals) if !isa(frame.locals[i], Nothing) - push!(local_vars, frame.code.slotnames[i]) + push!(local_vars, frame.code.code.slotnames[i]) push!(local_vals, QuoteNode(something(frame.locals[i]))) end end + ismeth = frame.code.scope isa Method for i = 1:length(frame.sparams) - push!(local_vars, frame.meth.sparam_syms[i]) + ismeth && push!(local_vars, frame.code.scope.sparam_syms[i]) push!(local_vals, QuoteNode(frame.sparams[i])) end res = gensym() @@ -136,11 +253,11 @@ function DebuggerFramework.eval_code(state, frame::JuliaStackFrame, command) Expr(:(=), res, expr), Expr(:tuple, res, Expr(:tuple, local_vars...)) )) - eval_res, res = Core.eval(frame.meth.module, eval_expr) + eval_res, res = Core.eval(moduleof(frame), eval_expr) j = 1 for i = 1:length(frame.locals) if !isa(frame.locals[i], Nothing) - frame.locals[i] = Some(res[j]) + frame.locals[i] = Some{Any}(res[j]) j += 1 end end @@ -157,13 +274,13 @@ end function DebuggerFramework.print_next_state(io::IO, state, frame::JuliaStackFrame) print(io, "About to run: ") - expr = pc_expr(frame, frame.pc) + expr = plain(pc_expr(frame, frame.pc[])) isa(expr, Expr) && (expr = copy(expr)) if isexpr(expr, :(=)) expr = expr.args[2] end if isexpr(expr, :call) || isexpr(expr, :return) - expr.args = map(var->maybe_quote(lookup_var_if_var(frame, var)), expr.args) + expr.args = map(var->maybe_quote(@eval_rhs(true, frame, var, frame.pc[])), expr.args) end if isa(expr, Expr) for (i, arg) in enumerate(expr.args) @@ -228,22 +345,14 @@ function DebuggerFramework.language_specific_prompt(state, frame::JuliaStackFram return julia_prompt end -function JuliaStackFrame(meth::Method) - JuliaStackFrame(meth, Vector{Any}(), - Vector{Any}(), Vector{Any}(), Vector{Any}(), - Dict{Symbol, Int}(), false, false, true) -end - function DebuggerFramework.debug(meth::Method, args...) - stack = [JuliaStackFrame(meth)] + stack = [enter_call(meth, args...)] DebuggerFramework.RunDebugger(stack) end function to_function(x) - if isa(x, Function) || isa(x, Core.IntrinsicFunction) - x - elseif isa(x, GlobalRef) - eval(x) + if isa(x, GlobalRef) + getfield(x.mod, x.name) else x end @@ -252,7 +361,7 @@ end _Typeof(x) = isa(x,Type) ? Type{x} : typeof(x) function is_function_def(ex) - (isa(ex,Expr) && ex.head == :(=) && isexpr(ex.args[1],:call)) || + (isexpr(ex, :(=)) && isexpr(ex.args[1], :call)) || isexpr(ex,:function) end @@ -260,56 +369,158 @@ function is_generated(meth) isdefined(meth, :generator) end -function determine_method_for_expr(expr; enter_generated = false) - f = to_function(expr.args[1]) - allargs = expr.args - # Extract keyword args - local kwargs = Expr(:parameters) - if length(allargs) > 1 && isexpr(allargs[2], :parameters) - kwargs = splice!(allargs, 2) - end - if !isempty(kwargs.args) +kwpair(ex::Expr) = [ex.args[1]; ex.args[2]] +kwpair(pr::Pair) = [pr.first; pr.second] + +""" + frun, allargs = prepare_args(fcall, fargs, kwargs) + +Prepare the complete argument sequence for a call to `fcall`. `fargs = [fcall, args...]` is a list +containing both `fcall` (the `#self#` slot in lowered code) and the positional +arguments supplied to `fcall`. `kwargs` is a list of keyword arguments, supplied either as +list of expressions `:(kwname=kwval)` or pairs `:kwname=>kwval`. + +For non-keyword methods, `frun === fcall`, but for methods with keywords `frun` will be the +keyword-sorter function for `fcall`. + +# Example + +```jldoctest; setup=(using ASTInterpreter2; empty!(ASTInterpreter2.junk)) +julia> mymethod(x) = 1 +mymethod (generic function with 1 method) + +julia> mymethod(x, y; verbose=false) = nothing +mymethod (generic function with 2 methods) + +julia> ASTInterpreter2.prepare_args(mymethod, [mymethod, 15], ()) +(mymethod, Any[mymethod, 15]) + +julia> ASTInterpreter2.prepare_args(mymethod, [mymethod, 1, 2], [:verbose=>true]) +(getfield( Symbol("#kw##mymethod"))(), Any[#kw##mymethod(), Any[:verbose, true], mymethod, 1, 2]) +``` +""" +function prepare_args(f, allargs, kwargs) + if !isempty(kwargs) of = f f = Core.kwfunc(f) - allargs = [f,reduce(vcat,Any[[ex.args[1];ex.args[2]] for ex in kwargs.args]),of, + allargs = [f,reduce(vcat,Any[kwpair(ex) for ex in kwargs]),of, allargs[2:end]...] elseif f === Core._apply f = to_function(allargs[2]) allargs = Base.append_any((allargs[2],), allargs[3:end]...) end - # Can happen for thunks created by generated functions - if !isa(f, Core.Builtin) && !isa(f, Core.IntrinsicFunction) - args = allargs[2:end] - argtypes = Tuple{map(_Typeof,args)...} - method = try - which(f, argtypes) - catch err - @show typeof(f) - println(f) - println(argtypes) - rethrow(err) - end - argtypes = Tuple{_Typeof(f), argtypes.parameters...} - args = allargs - sig = method.sig - isa(method, TypeMapEntry) && (method = method.func) - # Get static parameters - (ti, lenv) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), - argtypes, sig)::SimpleVector + return f, allargs +end + +""" + framecode, frameargs, lenv, argtypes = prepare_call(f, allargs; enter_generated=false) + +Prepare all the information needed to execute lowered code for `f` given arguments `allargs`. +`f` and `allargs` are the outputs of [`prepare_args`](@ref). +For `@generated` methods, set `enter_generated=true` if you want to extract the lowered code +of the generator itself. + +On return `framecode` is the [`JuliaFrameCode`](@ref) of the method. +`frameargs` contains the actual arguments needed for executing this frame (for generators, +this will be the types of `allargs`); +`lenv` is the "environment", i.e., the static parameters for `f` given `allargs`. +`argtypes` is the `Tuple`-type for this specific call (equivalent to the signature of the `MethodInstance`). + +# Example + +```jldoctest; setup=(using ASTInterpreter2; empty!(ASTInterpreter2.junk)) +julia> mymethod(x::Vector{T}) where T = 1 +mymethod (generic function with 1 method) + +julia> framecode, frameargs, lenv, argtypes = ASTInterpreter2.prepare_call(mymethod, [mymethod, [1.0,2.0]]); + +julia> framecode +ASTInterpreter2.JuliaFrameCode(mymethod(x::Array{T,1}) where T in Main at none:1, CodeInfo( +1 ─ return 1 +), Core.TypeMapEntry[#undef], BitSet([]), false, false, true) + +julia> frameargs +2-element Array{Any,1}: + mymethod + [1.0, 2.0] + +julia> lenv +svec(Float64) + +julia> argtypes +Tuple{typeof(mymethod),Array{Float64,1}} +``` +""" +function prepare_call(f, allargs; enter_generated = false) + args = allargs[2:end] + argtypes = Tuple{map(_Typeof,args)...} + method = try + which(f, argtypes) + catch err + @show typeof(f) + println(f) + println(argtypes) + rethrow(err) + end + argtypes = Tuple{_Typeof(f), argtypes.parameters...} + args = allargs + sig = method.sig + isa(method, TypeMapEntry) && (method = method.func) + # Get static parameters + (ti, lenv::SimpleVector) = ccall(:jl_type_intersection_with_env, Any, (Any, Any), + argtypes, sig)::SimpleVector + enter_generated &= is_generated(method) + if is_generated(method) && !enter_generated + framecode = get(genframedict, (method, argtypes), nothing) + else + framecode = get(framedict, method, nothing) + end + if framecode === nothing if is_generated(method) && !enter_generated # If we're stepping into a staged function, we need to use - # the specialization, rather than stepping thorugh the + # the specialization, rather than stepping through the # unspecialized method. code = Core.Compiler.get_staged(Core.Compiler.code_for_method(method, argtypes, lenv, typemax(UInt), false)) + generator = false else if is_generated(method) - args = map(_Typeof, args) + args = Any[_Typeof(a) for a in args] code = get_source(method.generator) + generator = true else code = get_source(method) + generator = false end end - return code, method, args, lenv + framecode = JuliaFrameCode(method, code; generator=generator) + if is_generated(method) && !enter_generated + genframedict[(method, argtypes)] = framecode + else + framedict[method] = framecode + end + end + return framecode, args, lenv, argtypes +end + +""" + framecode, frameargs, lenv, argtypes = determine_method_for_expr(expr; enter_generated = false) + +Prepare all the information needed to execute a particular `:call` expression `expr`. +For example, try `ASTInterpreter2.determine_method_for_expr(:(sum([1,2])))`. +See [`ASTInterpreter2.prepare_call`](@ref) for information about the outputs. +""" +function determine_method_for_expr(expr; enter_generated = false) + f = to_function(expr.args[1]) + allargs = expr.args + # Extract keyword args + local kwargs = Expr(:parameters) + if length(allargs) > 1 && isexpr(allargs[2], :parameters) + kwargs = splice!(allargs, 2) + end + f, allargs = prepare_args(f, allargs, kwargs.args) + # Can happen for thunks created by generated functions + if !isa(f, Core.Builtin) && !isa(f, Core.IntrinsicFunction) + return prepare_call(f, allargs; enter_generated=enter_generated) end nothing end @@ -328,78 +539,306 @@ function get_source(g::GeneratedFunctionStub) return eval(b) end -function Base.deepcopy_internal(x::LineInfoNode, stackdict::IdDict) - if haskey(stackdict, x) - return stackdict[x] +function copy_codeinfo(code::CodeInfo) + newcode = ccall(:jl_new_struct_uninit, Any, (Any,), CodeInfo)::CodeInfo + for (i, name) in enumerate(fieldnames(CodeInfo)) + if isdefined(code, name) + val = getfield(code, name) + ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), newcode, i-1, val===nothing ? val : copy(val)) + end end - deeper(x) = deepcopy_internal(x, stackdict) - stackdict[x] = LineInfoNode(x.mod, deeper(x.method), - deeper(x.file), deeper(x.line), deeper(x.inlined_at)) + return newcode end -function copy_codeinfo(code::CodeInfo) - old_code = code.code - code.code = UInt8[] - new_codeinfo = deepcopy(code) - new_codeinfo.code = old_code - code.code = old_code - new_codeinfo -end - -function prepare_locals(meth, code, argvals = (), generator = false) - code = copy_codeinfo(code) - # Construct the environment from the arguments - argnames = code.slotnames[1:meth.nargs] - locals = Array{Any}(undef, length(code.slotflags)) - ng = isa(code.ssavaluetypes, Int) ? code.ssavaluetypes : length(code.ssavaluetypes) - ssavalues = Array{Any}(undef, ng) - sparams = Array{Any}(undef, length(meth.sparam_syms)) +const calllike = Set([:call, :struct_type]) + +function extract_inner_call!(stmt, idx, once::Bool=false) + isa(stmt, Expr) || return nothing + once |= stmt.head ∈ calllike + for (i, a) in enumerate(stmt.args) + isa(a, Expr) || continue + ret = extract_inner_call!(a, idx, once) # doing this first extracts innermost calls + ret !== nothing && return ret + iscalllike = a.head ∈ calllike + if once && iscalllike + stmt.args[i] = NewSSAValue(idx) + return a + end + end + return nothing +end + +function replace_ssa!(stmt, ssalookup) + isa(stmt, Expr) || return nothing + for (i, a) in enumerate(stmt.args) + if isa(a, SSAValue) + stmt.args[i] = SSAValue(ssalookup[a.id]) + elseif isa(a, NewSSAValue) + stmt.args[i] = SSAValue(a.id) + else + replace_ssa!(a, ssalookup) + end + end + return nothing +end + +function lookup_global_refs!(ex::Expr) + for (i, a) in enumerate(ex.args) + if isa(a, GlobalRef) + r = getfield(a.mod, a.name) + ex.args[i] = QuoteNode(r) + elseif isa(a, Expr) + lookup_global_refs!(a) + end + end +end + +""" + optimize!(code::CodeInfo, mod::Module) + +Perform minor optimizations on the lowered AST in `code` to reduce execution time +of the interpreter. +Currently it looks up `GlobalRef`s (for which it needs `mod` to know the scope in +which this will run) and ensures that no statement includes nested `:call` expressions +(splitting them out into multiple SSA-form statements if needed). +""" +function optimize!(code::CodeInfo, mod::Module) + code.inferred && error("optimization of inferred code not implemented") + # TODO: because of builtins.jl, for CodeInfos like + # %1 = Core.apply_type + # %2 = (%1)(args...) + # it would be best to *not* resolve the GlobalRef at %1 + + ## Replace GlobalRefs with QuoteNodes + for (i, stmt) in enumerate(code.code) + if isa(stmt, GlobalRef) + code.code[i] = QuoteNode(getfield(stmt.mod, stmt.name)) + elseif isa(stmt, Expr) + if stmt.head == :call && isa(stmt.args[1], GlobalRef) + # Special handling of cglobal, which requires constants for its arguments + r = stmt.args[1]::GlobalRef + f = getfield(r.mod, r.name) + if f === Base.cglobal + code.code[i] = QuoteNode(Core.eval(mod, stmt)) + else + lookup_global_refs!(stmt) + end + else + lookup_global_refs!(stmt) + end + end + end + + ## Un-nest :call expressions (so that there will be only one :call per line) + # This will allow us to re-use args-buffers rather than having to allocate new ones each time. + old_code, old_codelocs = code.code, code.codelocs + code.code = new_code = eltype(old_code)[] + code.codelocs = new_codelocs = Int32[] + ssainc = fill(1, length(old_code)) + for (i, stmt) in enumerate(old_code) + loc = old_codelocs[i] + inner = extract_inner_call!(stmt, length(new_code)+1) + while inner !== nothing + push!(new_code, inner) + push!(new_codelocs, loc) + ssainc[i] += 1 + inner = extract_inner_call!(stmt, length(new_code)+1) + end + push!(new_code, stmt) + push!(new_codelocs, loc) + end + # Fix all the SSAValues and GotoNodes + ssalookup = cumsum(ssainc) + for (i, stmt) in enumerate(new_code) + if isa(stmt, GotoNode) + new_code[i] = GotoNode(ssalookup[stmt.label]) + elseif isa(stmt, SSAValue) + new_code[i] = SSAValue(ssalookup[stmt.id]) + elseif isa(stmt, NewSSAValue) + new_code[i] = SSAValue(stmt.id) + elseif isa(stmt, Expr) + replace_ssa!(stmt, ssalookup) + if stmt.head == :gotoifnot && isa(stmt.args[2], Int) + stmt.args[2] = ssalookup[stmt.args[2]] + end + end + end + code.ssavaluetypes = length(new_code) + return code +end + +plain(stmt) = stmt + +function prepare_locals(framecode, argvals::Vector{Any}) + meth, code = framecode.scope::Method, framecode.code + ssavt = code.ssavaluetypes + ng = isa(ssavt, Int) ? ssavt : length(ssavt::Vector{Any}) + nargs = length(argvals) + if !isempty(junk) + oldframe = pop!(junk) + locals, ssavalues, sparams = oldframe.locals, oldframe.ssavalues, oldframe.sparams + exception_frames, last_reference = oldframe.exception_frames, oldframe.last_reference + callargs = oldframe.callargs + last_exception, pc = oldframe.last_exception, oldframe.pc + resize!(locals, length(code.slotflags)) + resize!(ssavalues, ng) + resize!(sparams, length(meth.sparam_syms)) + empty!(exception_frames) + empty!(last_reference) + last_exception[] = nothing + pc[] = JuliaProgramCounter(1) + else + locals = Vector{Union{Nothing,Some{Any}}}(undef, length(code.slotflags)) + ssavalues = Vector{Any}(undef, ng) + sparams = Vector{Any}(undef, length(meth.sparam_syms)) + exception_frames = Int[] + last_reference = Dict{Symbol,Int}() + callargs = Any[] + last_exception = Ref{Any}(nothing) + pc = Ref(JuliaProgramCounter(1)) + end for i = 1:meth.nargs - if meth.isva && i == length(argnames) - locals[i] = length(argvals) >= i ? Some(tuple(argvals[i:end]...)) : Some(()) + if meth.isva && i == meth.nargs + locals[i] = nargs < i ? Some{Any}(()) : (let i=i; Some{Any}(ntuple(k->argvals[i+k-1], nargs-i+1)); end) break end - locals[i] = length(argvals) >= i ? Some(argvals[i]) : Some(()) + locals[i] = nargs >= i ? Some{Any}(argvals[i]) : Some{Any}(()) end # add local variables initially undefined for i = (meth.nargs+1):length(code.slotnames) locals[i] = nothing end - used = find_used(code) - JuliaStackFrame(meth, code, locals, ssavalues, used, sparams, Int[], nothing, - JuliaProgramCounter(1), Dict{Symbol,Int}(), false, generator, - true) + JuliaStackFrame(framecode, locals, ssavalues, sparams, exception_frames, last_exception, + pc, last_reference, callargs) end +""" + frame = build_frame(framecode::JuliaFrameCode, frameargs, lenv) + +Construct a new `JuliaStackFrame` for `framecode`, given lowered-code arguments `frameargs` and +static parameters `lenv`. See [`ASTInterpreter2.prepare_call`](@ref) for information about how to prepare the inputs. +""" +function build_frame(framecode, args, lenv) + frame = prepare_locals(framecode, args) + # Add static parameters to environment + for i = 1:length(lenv) + frame.sparams[i] = lenv[i] + end + return frame +end + +""" + frame = enter_call_expr(expr; enter_generated=false) + +Build a `JuliaStackFrame` ready to execute the expression `expr`. Set `enter_generated=true` +if you want to execute the generator of a `@generated` function, rather than the code that +would be created by the generator. + +# Example + +```jldoctest; setup=(using ASTInterpreter2; empty!(ASTInterpreter2.junk)) +julia> mymethod(x) = x+1 +mymethod (generic function with 1 method) + +julia> ASTInterpreter2.enter_call_expr(:(\$mymethod(1))) +JuliaStackFrame(ASTInterpreter2.JuliaFrameCode(mymethod(x) in Main at none:1, CodeInfo( +1 ─ %1 = (\$(QuoteNode(+)))(x, 1) +└── return %1 +), Core.TypeMapEntry[#undef, #undef], BitSet([1]), false, false, true), Union{Nothing, Some{Any}}[Some(mymethod), Some(1)], Any[#undef, #undef], Any[], Int64[], Base.RefValue{Any}(nothing), Base.RefValue{ASTInterpreter2.JuliaProgramCounter}(JuliaProgramCounter(1)), Dict{Symbol,Int64}(), Any[]) +julia> mymethod(x::Vector{T}) where T = 1 +mymethod (generic function with 2 methods) + +julia> a = [1.0, 2.0] +2-element Array{Float64,1}: + 1.0 + 2.0 + +julia> ASTInterpreter2.enter_call_expr(:(\$mymethod(\$a))) +JuliaStackFrame(ASTInterpreter2.JuliaFrameCode(mymethod(x::Array{T,1}) where T in Main at none:1, CodeInfo( +1 ─ return 1 +), Core.TypeMapEntry[#undef], BitSet([]), false, false, true), Union{Nothing, Some{Any}}[Some(mymethod), Some([1.0, 2.0])], Any[#undef], Any[Float64], Int64[], Base.RefValue{Any}(nothing), Base.RefValue{ASTInterpreter2.JuliaProgramCounter}(JuliaProgramCounter(1)), Dict{Symbol,Int64}(), Any[]) +``` + +See [`enter_call`](@ref) for a similar approach not based on expressions. +""" function enter_call_expr(expr; enter_generated = false) r = determine_method_for_expr(expr; enter_generated = enter_generated) if r !== nothing - code, method, args, lenv = r - frame = prepare_locals(method, code, args, enter_generated) - # Add static parameters to environment - for i = 1:length(lenv) - frame.sparams[i] = lenv[i] - end - return frame + return build_frame(r[1:end-1]...) end nothing end +""" + frame = enter_call(f, args...; kwargs...) + +Build a `JuliaStackFrame` ready to execute `f` with the specified positional and keyword arguments. + +# Example + +```jldoctest; setup=(using ASTInterpreter2; empty!(ASTInterpreter2.junk)) +julia> mymethod(x) = x+1 +mymethod (generic function with 1 method) + +julia> ASTInterpreter2.enter_call(mymethod, 1) +JuliaStackFrame(ASTInterpreter2.JuliaFrameCode(mymethod(x) in Main at none:1, CodeInfo( +1 ─ %1 = ($(QuoteNode(+)))(x, 1) +└── return %1 +), Core.TypeMapEntry[#undef, #undef], BitSet([1]), false, false, true), Union{Nothing, Some{Any}}[Some(mymethod), Some(1)], Any[#undef, #undef], Any[], Int64[], Base.RefValue{Any}(nothing), Base.RefValue{ASTInterpreter2.JuliaProgramCounter}(JuliaProgramCounter(1)), Dict{Symbol,Int64}(), Any[]) + +julia> mymethod(x::Vector{T}) where T = 1 +mymethod (generic function with 2 methods) + +julia> ASTInterpreter2.enter_call(mymethod, [1.0, 2.0]) +JuliaStackFrame(ASTInterpreter2.JuliaFrameCode(mymethod(x::Array{T,1}) where T in Main at none:1, CodeInfo( +1 ─ return 1 +), Core.TypeMapEntry[#undef], BitSet([]), false, false, true), Union{Nothing, Some{Any}}[Some(mymethod), Some([1.0, 2.0])], Any[#undef], Any[Float64], Int64[], Base.RefValue{Any}(nothing), Base.RefValue{ASTInterpreter2.JuliaProgramCounter}(JuliaProgramCounter(1)), Dict{Symbol,Int64}(), Any[]) +``` + +For a `@generated` function you can use `enter_call((f, true), args...; kwargs...)` +to execute the generator of a `@generated` function, rather than the code that +would be created by the generator. + +See [`enter_call_expr`](@ref) for a similar approach based on expressions. +""" +function enter_call(@nospecialize(finfo), @nospecialize(args...); kwargs...) + if isa(finfo, Tuple) + f = finfo[1] + enter_generated = finfo[2]::Bool + else + f = finfo + enter_generated = false + end + f, allargs = prepare_args(f, Any[f, args...], kwargs) + # Can happen for thunks created by generated functions + if isa(f, Core.Builtin) || isa(f, Core.IntrinsicFunction) + error(f, " is a builtin or intrinsic") + end + r = prepare_call(f, allargs; enter_generated=enter_generated) + if r !== nothing + return build_frame(r[1:end-1]...) + end + return nothing +end + function maybe_step_through_wrapper!(stack) - last = stack[1].code.code[end-1] + length(stack[1].code.code.code) < 2 && return stack + last = plain(stack[1].code.code.code[end-1]) isexpr(last, :(=)) && (last = last.args[2]) - is_kw = startswith(String(Base.unwrap_unionall(stack[1].meth.sig).parameters[1].name.name), "#kw") + stack1 = stack[1] + is_kw = stack1.code.scope isa Method && startswith(String(Base.unwrap_unionall(stack1.code.scope.sig).parameters[1].name.name), "#kw") if is_kw || isexpr(last, :call) && any(x->x==SlotNumber(1), last.args) - # If the last expr calls #self# or passes it to an implemetnation method, - # this is a wrapper function that we might want to step though - frame = stack[1] - pc = frame.pc - while pc != JuliaProgramCounter(length(frame.code.code)-1) - pc = next_call!(frame, pc) + # If the last expr calls #self# or passes it to an implementation method, + # this is a wrapper function that we might want to step through + frame = stack1 + pc = frame.pc[] + while pc != JuliaProgramCounter(length(frame.code.code.code)-1) + pc = next_call!(Compiled(), frame, pc) end - stack[1] = JuliaStackFrame(frame, pc; wrapper=true) - pushfirst!(stack, enter_call_expr(Expr(:call, map(x->lookup_var_if_var(frame, x), last.args)...))) + stack[1] = JuliaStackFrame(JuliaFrameCode(frame.code; wrapper=true), frame, pc) + newcall = Expr(:call, map(x->@eval_rhs(true, frame, x, pc), last.args)...) + pushfirst!(stack, enter_call_expr(newcall)) return maybe_step_through_wrapper!(stack) end stack @@ -463,7 +902,7 @@ function _make_stack(mod, arg) theargs = $(esc(args)) stack = [ASTInterpreter2.enter_call_expr(Expr(:call,theargs...))] ASTInterpreter2.maybe_step_through_wrapper!(stack) - stack[1] = ASTInterpreter2.JuliaStackFrame(stack[1], ASTInterpreter2.maybe_next_call!(stack[1])) + stack[1] = ASTInterpreter2.JuliaStackFrame(stack[1], ASTInterpreter2.maybe_next_call!(Compiled(), stack[1])) stack end end @@ -480,6 +919,42 @@ macro enter(arg) end end +""" + @interpret f(args; kwargs...) + +Evaluate `f` on the specified arguments using the interpreter. + +# Example + +```jldoctest; setup=(using ASTInterpreter2; empty!(ASTInterpreter2.junk)) +julia> a = [1, 7] +2-element Array{Int64,1}: + 1 + 7 + +julia> sum(a) +8 + +julia> @interpret sum(a) +8 +``` +""" +macro interpret(arg) + args = try + extract_args(__module__, arg) + catch e + return :(throw($e)) + end + quote + theargs = $(esc(args)) + stack = JuliaStackFrame[] + frame = ASTInterpreter2.enter_call_expr(Expr(:call,theargs...)) + empty!(framedict) # start fresh each time; kind of like bumping the world age at the REPL prompt + empty!(genframedict) + finish_and_return!(stack, frame) + end +end + include("commands.jl") end # module diff --git a/src/commands.jl b/src/commands.jl index eb6a402..84d09b1 100644 --- a/src/commands.jl +++ b/src/commands.jl @@ -2,28 +2,28 @@ function perform_return!(state) returning_frame = state.stack[1] returning_expr = pc_expr(returning_frame) @assert isexpr(returning_expr, :return) - val = lookup_var_if_var(returning_frame, returning_expr.args[1]) + val = @eval_rhs(true, returning_frame, returning_expr.args[1], returning_frame.pc[]) if length(state.stack) != 1 calling_frame = state.stack[2] - if returning_frame.generator + if returning_frame.code.generator # Don't do anything here, just return us to where we were else - prev = pc_expr(calling_frame) + prev = plain(pc_expr(calling_frame)) if isexpr(prev, :(=)) do_assignment!(calling_frame, prev.args[1], val) elseif isassign(calling_frame) - do_assignment!(calling_frame, getlhs(calling_frame.pc), val) + do_assignment!(calling_frame, getlhs(calling_frame.pc[]), val) end - state.stack[2] = JuliaStackFrame(calling_frame, maybe_next_call!(calling_frame, - calling_frame.pc + 1)) + state.stack[2] = JuliaStackFrame(calling_frame, maybe_next_call!(Compiled(), calling_frame, + calling_frame.pc[] + 1)) end else - @assert !returning_frame.generator + @assert !returning_frame.code.generator state.overall_result = val end popfirst!(state.stack) - if !isempty(state.stack) && state.stack[1].wrapper - state.stack[1] = JuliaStackFrame(state.stack[1], finish!(state.stack[1])) + if !isempty(state.stack) && state.stack[1].code.wrapper + state.stack[1] = JuliaStackFrame(state.stack[1], finish!(Compiled(), state.stack[1])) perform_return!(state) end end @@ -47,12 +47,12 @@ end function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, ::Union{Val{:nc},Val{:n},Val{:se}}, command) pc = try - command == "nc" ? next_call!(frame) : - command == "n" ? next_line!(frame, state.stack) : - #= command == "se" =# step_expr(frame) + command == "nc" ? next_call!(Compiled(), frame) : + command == "n" ? next_line!(Compiled(), frame, state.stack) : + #= command == "se" =# step_expr!(Compiled(), frame) catch err propagate_exception!(state, err) - state.stack[1] = JuliaStackFrame(state.stack[1], next_call!(state.stack[1], state.stack[1].pc)) + state.stack[1] = JuliaStackFrame(state.stack[1], next_call!(Compiled(), state.stack[1], state.stack[1].pc[])) return true end if pc != nothing @@ -64,15 +64,15 @@ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, ::Unio end function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, cmd::Union{Val{:s},Val{:si},Val{:sg}}, command) - pc = frame.pc + pc = frame.pc[] first = true while true - expr = pc_expr(frame, pc) + expr = plain(pc_expr(frame, pc)) if isa(expr, Expr) if is_call(expr) isexpr(expr, :(=)) && (expr = expr.args[2]) args = map(x->isa(x, QuoteNode) ? x.value : - lookup_var_if_var(frame, x), expr.args) + @eval_rhs(true, frame, x, pc), expr.args) expr = Expr(:call, args...) f = (expr.args[1] == Core._apply) ? expr.args[2] : expr.args[1] ok = true @@ -80,10 +80,10 @@ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, cmd::U new_frame = enter_call_expr(expr; enter_generated = command == "sg") if (cmd == Val{:s}() || cmd == Val{:sg}()) - new_frame = JuliaStackFrame(new_frame, maybe_next_call!(new_frame)) + new_frame = JuliaStackFrame(new_frame, maybe_next_call!(Compiled(), new_frame)) end # Don't step into Core.Compiler - if new_frame.meth.module == Core.Compiler + if moduleof(new_frame) == Core.Compiler ok = false else state.stack[1] = JuliaStackFrame(frame, pc) @@ -96,7 +96,7 @@ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, cmd::U if !ok # It's confusing if we step into the next call, so just go there # and then return - state.stack[1] = JuliaStackFrame(frame, next_call!(frame, pc)) + state.stack[1] = JuliaStackFrame(frame, next_call!(Compiled(), frame, pc)) return true end elseif !first && isexpr(expr, :return) @@ -107,10 +107,10 @@ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, cmd::U first = false command == "si" && break new_pc = try - _step_expr(frame, pc) + _step_expr!(Compiled(), frame, pc) catch err propagate_exception!(state, err) - state.stack[1] = JuliaStackFrame(state.stack[1], next_call!(state.stack[1], pc)) + state.stack[1] = JuliaStackFrame(state.stack[1], next_call!(Compiled(), state.stack[1], pc)) return true end if new_pc == nothing @@ -126,7 +126,7 @@ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, cmd::U end function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, ::Val{:finish}, cmd) - state.stack[1] = JuliaStackFrame(frame, finish!(frame)) + state.stack[1] = JuliaStackFrame(frame, finish!(Compiled(), frame)) perform_return!(state) return true end @@ -135,12 +135,12 @@ end Runs code_typed on the call we're about to run """ function DebuggerFramework.execute_command(state, frame::JuliaStackFrame, ::Val{:code_typed}, cmd) - expr = pc_expr(frame, frame.pc) + expr = plain(pc_expr(frame, frame.pc[])) if isa(expr, Expr) if is_call(expr) isexpr(expr, :(=)) && (expr = expr.args[2]) args = map(x->isa(x, QuoteNode) ? x.value : - lookup_var_if_var(frame, x), expr.args) + @eval_rhs(true, frame, x, frame.pc[]), expr.args) f = args[1] if f == Core._apply f = to_function(args[2]) diff --git a/src/generate_builtins.jl b/src/generate_builtins.jl new file mode 100644 index 0000000..e4e7e3a --- /dev/null +++ b/src/generate_builtins.jl @@ -0,0 +1,141 @@ +# This file generates builtins.jl. + +# Look up the expected number of arguments in Core.Compiler.tfunc data +function generate_fcall(f, table, id) + if id !== nothing + minarg, maxarg, tfunc = table[id] + else + minarg = 0 + maxarg = typemax(Int) + end + # The tfunc tables are wrong for fptoui and fptosi + if f == "Base.fptoui" || f == "Base.fptosi" + minarg = 2 + end + # Generate a separate call for each number of arguments + if maxarg < typemax(Int) + wrapper = minarg == maxarg ? "" : "if nargs == " + for nargs = minarg:maxarg + if minarg < maxarg + wrapper *= "$nargs\n " + end + argcall = "" + for i = 1:nargs + argcall *= "@eval_rhs(true, frame, args[$(i+1)])" + if i < nargs + argcall *= ", " + end + end + wrapper *= "return Some{Any}($f($argcall))" + if nargs < maxarg + wrapper *= "\n elseif nargs == " + end + end + if minarg < maxarg + wrapper *= "\n end" + end + return wrapper + end + # A built-in with arbitrary or unknown number of arguments. + # This will (unfortunately) use dynamic dispatch. + return "return Some{Any}($f(getargs(args, frame)...))" +end + +# `io` is for the generated source file +# `intrinsicsfile` is the path to Julia's `src/intrinsics.h` file +function generate_builtins(io::IO) + pat = r"(ADD_I|ALIAS)\((\w*)," + print(io, +""" +# This file is generated by `generate_builtins.jl`. Do not edit by hand. + +function getargs(args, frame) + nargs = length(args)-1 # skip f + callargs = resize!(frame.callargs, nargs) + for i = 1:nargs + callargs[i] = @eval_rhs(true, frame, args[i+1]) + end + return callargs +end + +\"\"\" + ret = maybe_evaluate_builtin(frame, call_expr) + +If `call_expr` is to a builtin function, evaluate it, returning the result inside a `Some` wrapper. +Otherwise, return `call_expr`. +\"\"\" +function maybe_evaluate_builtin(frame, call_expr) + # By having each call appearing statically in the "switch" block below, + # each gets call-site optimized. + + args = call_expr.args + nargs = length(args) - 1 + fex = args[1] + if isa(fex, QuoteNode) + f = fex.value + else + f = @eval_rhs(true, frame, fex) + end + # Builtins and intrinsics have empty method tables. We can circumvent + # a long "switch" check by looking for this. + mt = typeof(f).name.mt + if isa(mt, Core.MethodTable) + isempty(mt) || return call_expr + end + # Builtins +""") + firstcall = true + for ft in subtypes(Core.Builtin) + ft === Core.IntrinsicFunction && continue + ft === getfield(Core, Symbol("#kw##invoke")) && continue # handle this one later + head = firstcall ? "if" : "elseif" + firstcall = false + f = ft.instance + # Tuple is common, especially for returned values from calls. It's worth avoiding + # dynamic dispatch through a call to `ntuple`. + if f === tuple + print(io, +""" + $head f === $f + return Some{Any}(ntuple(i->@eval_rhs(true, frame, args[i+1]), length(args)-1)) +""") + continue + end + id = findfirst(isequal(f), Core.Compiler.T_FFUNC_KEY) + fcall = generate_fcall(f, Core.Compiler.T_FFUNC_VAL, id) + print(io, +""" + $head f === $f + $fcall +""") + firstcall = false + end + print(io, +""" + # Intrinsics +""") + for fsym in names(Core.Intrinsics) + fsym == :Intrinsics && continue + isdefined(Base, fsym) || (println("skipping ", fname); continue) + f = getfield(Base, fsym) + f isa Core.IntrinsicFunction || error("not an intrinsic") + id = reinterpret(Int32, f) + 1 + f = isdefined(Base, fsym) ? "Base.$fsym" : + isdefined(Core, fsym) ? "Core.$fsym" : error("whoops on $f") + fcall = generate_fcall(f, Core.Compiler.T_IFUNC, id) + print(io, +""" + elseif f === $f + $fcall +""") + end + print(io, +""" + end + if isa(f, getfield(Core, Symbol("#kw##invoke"))) + return Some{Any}(getfield(Core, Symbol("#kw##invoke"))(getargs(args, frame)...)) + end + return call_expr +end +""") +end diff --git a/src/interpret.jl b/src/interpret.jl index f4be9b0..30366aa 100644 --- a/src/interpret.jl +++ b/src/interpret.jl @@ -2,97 +2,174 @@ getlhs(pc) = SSAValue(pc.next_stmt) -isassign(fr) = isassign(fr, fr.pc) -isassign(fr, pc) = (pc.next_stmt in fr.used) +isassign(fr) = isassign(fr, fr.pc[]) +isassign(fr, pc) = (pc.next_stmt in fr.code.used) -lookup_var(frame, val::SSAValue) = frame.ssavalues[val.id+1] +lookup_var(frame, val::SSAValue) = frame.ssavalues[val.id] lookup_var(frame, ref::GlobalRef) = getfield(ref.mod, ref.name) -lookup_var(frame, slot::SlotNumber) = something(frame.locals[slot.id]) -function lookup_var(frame, e::Expr) - isexpr(e, :the_exception) && return frame.last_exception[] - isexpr(e, :boundscheck) && return true - isexpr(e, :static_parameter) || error() - frame.sparams[e.args[1]] +function lookup_var(frame, slot::SlotNumber) + val = frame.locals[slot.id] + val !== nothing && return val.value + error("slot ", slot, " not assigned") end -function finish!(frame) - pc = frame.pc - while true - new_pc = _step_expr(frame, pc) - new_pc == nothing && return pc - pc = new_pc +function lookup_expr(frame, e::Expr, fallback::Bool) + head = e.head + head == :the_exception && return frame.last_exception[] + head == :boundscheck && return true + head == :static_parameter && return frame.sparams[e.args[1]::Int] + return fallback ? e : error() +end + +""" + rhs = @eval_rhs(flag, frame, node) + rhs = @eval_rhs(flag, frame, node, pc) + +This macro substitutes for a function call, as a performance optimization to avoid dynamic dispatch. +It calls `lookup_var(frame, node)` when appropriate, otherwise: + +* with `flag=false` it throws an error (if `lookup_var` didn't already handle the call) +* with `flag=true` it will additionally try `lookup_expr`, resolve `QuoteNode`s, and otherwise return `node` +* with `flag=Compiled()` it will additionally recurse into calls using Julia's normal compiled-code evaluation +* with `flag=stack`, a vector of `JuliaStackFrames`, it will recurse via the interpreter + +If `flag` isn't a Bool you need to supply `pc`. +""" +macro eval_rhs(flag, frame, rest...) + node = rest[1] + pc = length(rest) == 1 ? nothing : rest[2] + nodetmp = gensym(:node) # used to hoist, e.g., args[4] + if flag == true + fallback = quote + isa($nodetmp, QuoteNode) ? $nodetmp.value : + isa($nodetmp, Expr) ? lookup_expr($(esc(frame)), $nodetmp, true) : $nodetmp + end + elseif flag == false + fallback = :(error("bad node ", $nodetmp)) + else + fallback = quote + isa($nodetmp, QuoteNode) ? $nodetmp.value : + isa($nodetmp, Expr) ? eval_rhs($(esc(flag)), $(esc(frame)), $nodetmp, $(esc(pc))) : + $nodetmp + end + end + quote + $nodetmp = $(esc(node)) + isa($nodetmp, SSAValue) ? lookup_var($(esc(frame)), $nodetmp) : + isa($nodetmp, GlobalRef) ? lookup_var($(esc(frame)), $nodetmp) : + isa($nodetmp, SlotNumber) ? lookup_var($(esc(frame)), $nodetmp) : + $fallback end end instantiate_type_in_env(arg, spsig, spvals) = ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), arg, spsig, spvals) -function evaluate_call(frame, call_expr) - args = Array{Any}(undef, length(call_expr.args)) +function collect_args(frame, call_expr) + args = frame.callargs + resize!(args, length(call_expr.args)) for i = 1:length(args) - arg = call_expr.args[i] - if isa(arg, QuoteNode) - args[i] = arg.value - elseif isa(arg, Union{SSAValue, GlobalRef, Slot}) - args[i] = lookup_var(frame, arg) - elseif isexpr(arg, :&) - args[i] = Expr(:&, lookup_var(frame, arg.args[1])) - elseif isa(arg, Expr) - args[i] = eval_rhs(frame, arg) - else - args[i] = arg - end + args[i] = @eval_rhs(true, frame, call_expr.args[i]) + end + return args +end + +""" + ret = evaluate_foreigncall!(stack, frame::JuliaStackFrame, call_expr, pc) + +Evaluate a `:foreigncall` (from a `ccall`) statement `callexpr` in the context of `frame`. +`stack` and `pc` are unused, but supplied for consistency with [`evaluate_call!`](@ref). +""" +function evaluate_foreigncall!(stack, frame::JuliaStackFrame, call_expr::Expr, pc) + args = collect_args(frame, call_expr) + for i = 1:length(args) + arg = args[i] + args[i] = isa(arg, Symbol) ? QuoteNode(arg) : arg end - # Don't go through eval since this may have unqouted, symbols and + scope = frame.code.scope + if !isempty(frame.sparams) && scope isa Method + sig = scope.sig + args[2] = instantiate_type_in_env(args[2], sig, frame.sparams) + args[3] = Core.svec(map(args[3]) do arg + instantiate_type_in_env(arg, sig, frame.sparams) + end...) + end + return Core.eval(moduleof(frame), Expr(:foreigncall, args...)) +end + +function evaluate_call!(::Compiled, frame::JuliaStackFrame, call_expr::Expr, pc) + ret = maybe_evaluate_builtin(frame, call_expr) + isa(ret, Some{Any}) && return ret.value + fargs = collect_args(frame, call_expr) + # Don't go through eval since this may have unquoted symbols and # exprs - if isexpr(call_expr, :foreigncall) - args = map(args) do arg - isa(arg, Symbol) ? QuoteNode(arg) : arg - end - if !isempty(frame.sparams) - args[2] = instantiate_type_in_env(args[2], frame.meth.sig, frame.sparams) - args[3] = Core.svec(map(args[3]) do arg - instantiate_type_in_env(arg, frame.meth.sig, frame.sparams) - end...) - end - ret = Core.eval(frame.meth.module, Expr(:foreigncall, args...)) + f = to_function(fargs[1]) + if isa(f, CodeInfo) + error("CodeInfo") + ret = finish_and_return!(Compiled(), enter_call_expr(frame, call_expr)) else - f = to_function(args[1]) - if isa(f, CodeInfo) - ret = finish!(enter_call_expr(frame, call_expr)) - else - # Don't go through eval since this may have unqouted, symbols and - # exprs - ret = f(args[2:end]...) - end + popfirst!(fargs) # now it's really just `args` + ret = f(fargs...) end return ret end -function do_assignment!(frame, lhs, rhs) +function evaluate_call!(stack, frame::JuliaStackFrame, call_expr::Expr, pc) + ret = maybe_evaluate_builtin(frame, call_expr) + isa(ret, Some{Any}) && return ret.value + fargs = collect_args(frame, call_expr) + framecode, lenv = get_call_framecode(fargs, frame.code, pc.next_stmt) + if lenv === nothing + return framecode # this was a Builtin + end + frame.pc[] = pc # to mark position in the frame (e.g., if we hit breakpoint or error) + push!(stack, frame) + newframe = build_frame(framecode, fargs, lenv) + ret = finish_and_return!(stack, newframe) + pop!(stack) + push!(junk, newframe) # rather than going through GC, just re-use it + return ret +end + +""" + ret = evaluate_call!(Compiled(), frame::JuliaStackFrame, call_expr, pc) + ret = evaluate_call!(stack, frame::JuliaStackFrame, call_expr, pc) + +Evaluate a `:call` expression `call_expr` in the context of `frame`. +The first causes it to be executed using Julia's normal dispatch (compiled code), +whereas the second recurses in via the interpreter. `stack` should be a vector of [`JuliaStackFrame`](@ref). +""" +evaluate_call! + +function do_assignment!(frame, @nospecialize(lhs), @nospecialize(rhs)) if isa(lhs, SSAValue) - frame.ssavalues[lhs.id+1] = rhs - elseif isa(lhs, Slot) - frame.locals[lhs.id] = Some(rhs) - frame.last_reference[frame.code.slotnames[lhs.id]] = + frame.ssavalues[lhs.id] = rhs + elseif isa(lhs, SlotNumber) + frame.locals[lhs.id] = Some{Any}(rhs) + frame.last_reference[frame.code.code.slotnames[lhs.id]] = lhs.id elseif isa(lhs, GlobalRef) Base.eval(lhs.mod,:($(lhs.name) = $(QuoteNode(rhs)))) end end -eval_rhs(frame, node) = eval(node) -function eval_rhs(frame, node::Expr) - if isexpr(node, :new) - new_expr = Expr(:new, map(x->QuoteNode(lookup_var_if_var(frame, x)), - node.args)...) - rhs = Core.eval(frame.meth.module, new_expr) - elseif isexpr(node, :isdefined) +function eval_rhs(stack, frame, node::Expr, pc) + head = node.head + if head == :new + rhs = ccall(:jl_new_struct_uninit, Any, (Any,), @eval_rhs(true, frame, node.args[1])) + for i = 1:length(node.args) - 1 + ccall(:jl_set_nth_field, Cvoid, (Any, Csize_t, Any), rhs, i-1, @eval_rhs(true, frame, node.args[i+1])) + end + elseif head == :isdefined rhs = check_isdefined(frame, node.args[1]) + elseif head == :enter + rhs = length(frame.exception_frames) + elseif head == :call + rhs = evaluate_call!(stack, frame, node, pc) + elseif head == :foreigncall + rhs = evaluate_foreigncall!(stack, frame, node, pc) else - rhs = (isexpr(node, :call) || isexpr(node, :foreigncall)) ? - evaluate_call(frame, node) : - lookup_var_if_var(frame, node) + rhs = lookup_expr(frame, node, false) end if isa(rhs, QuoteNode) rhs = rhs.value @@ -100,48 +177,55 @@ function eval_rhs(frame, node::Expr) return rhs end -eval_rhs(frame, node::Union{SSAValue, GlobalRef, SlotNumber}) = lookup_var(frame, node) -eval_rhs(frame, node::QuoteNode) = node.value -check_isdefined(frame, node::Slot) = isdefined(frame.locals, slot.id) +check_isdefined(frame, node::Slot) = isassigned(frame.locals, slot.id) function check_isdefined(frame, node::Expr) - node.head == :static_parameter && return isdefined(frame.sparams, node.args[1]) + node.head == :static_parameter && return isassigned(frame.sparams, node.args[1]) end -function _step_expr(frame, pc) +function _step_expr!(stack, frame, pc) node = pc_expr(frame, pc) + local rhs try - if isassign(frame, pc) - lhs = getlhs(pc) - rhs = eval_rhs(frame, node) - do_assignment!(frame, lhs, rhs) - elseif isa(node, Expr) + if isa(node, Expr) if node.head == :(=) lhs = node.args[1] - rhs = eval_rhs(frame, node.args[2]) + rhs = @eval_rhs(stack, frame, node.args[2], pc) do_assignment!(frame, lhs, rhs) # Special case hack for readability. # ret = rhs - elseif node.head == :& elseif node.head == :gotoifnot - arg = eval_rhs(frame, node.args[1]) + arg = @eval_rhs(stack, frame, node.args[1], pc) if !isa(arg, Bool) - throw(TypeError(frame.meth.name, "if", Bool, node.args[1])) + throw(TypeError(nameof(frame), "if", Bool, node.args[1])) end if !arg return JuliaProgramCounter(node.args[2]) end - elseif node.head == :call || node.head == :foreigncall - evaluate_call(frame, node) - elseif node.head == :static_typeof - elseif node.head == :type_goto || node.head == :inbounds + elseif node.head == :call + rhs = evaluate_call!(stack, frame, node, pc) + elseif node.head == :foreigncall + rhs = evaluate_foreigncall!(stack, frame, node, pc) + elseif node.head == :new + rhs = eval_rhs(stack, frame, node, pc) + elseif node.head == :static_typeof || node.head == :type_goto + error(node, ", despite the docs, still exists") + elseif node.head == :meta || node.head == :inbounds || node.head == :simdloop elseif node.head == :enter - push!(frame.exception_frames, node.args[1]) + rhs = node.args[1] + push!(frame.exception_frames, rhs) elseif node.head == :leave for _ = 1:node.args[1] pop!(frame.exception_frames) end + elseif node.head == :pop_exception + n = lookup_var(frame, node.args[1]) + deleteat!(frame.exception_frames, n+1:length(frame.exception_frames)) + elseif node.head == :isdefined + rhs = check_isdefined(frame, node.args[1]) elseif node.head == :static_parameter + rhs = frame.sparams[node.args[1]] elseif node.head == :gc_preserve_end || node.head == :gc_preserve_begin + rhs = @eval_rhs(true, frame, node.args[1]) elseif node.head == :return return nothing else @@ -150,31 +234,86 @@ function _step_expr(frame, pc) elseif isa(node, GotoNode) return JuliaProgramCounter(node.label) elseif isa(node, QuoteNode) - ret = node.value + rhs = node.value else - ret = eval_rhs(frame, node) + rhs = @eval_rhs(stack, frame, node, pc) end catch err isempty(frame.exception_frames) && rethrow(err) frame.last_exception[] = err return JuliaProgramCounter(frame.exception_frames[end]) end - return JuliaProgramCounter(pc.next_stmt + 1) + if isassign(frame, pc) + if !@isdefined(rhs) + @show frame node pc + end + lhs = getlhs(pc) + do_assignment!(frame, lhs, rhs) + end + return pc + 1 +end + +""" + pc = step_expr!(stack, frame) + +Execute the next statement in `frame`. `pc` is the new program counter, or `nothing` +if execution terminates. +`stack` controls call evaluation; `stack = Compiled()` evaluates :call expressions +by normal dispatch, whereas a vector of `JuliaStackFrame`s will use recursive interpretation. +""" +function step_expr!(stack, frame) + pc = _step_expr!(stack, frame, frame.pc[]) + pc === nothing && return nothing + frame.pc[] = pc +end + +""" + pc = finish!(stack, frame, pc=frame.pc[]) + +Run `frame` until execution terminates. `pc` is the program counter for the final statement. +`stack` controls call evaluation; `stack = Compiled()` evaluates :call expressions +by normal dispatch, whereas a vector of `JuliaStackFrame`s will use recursive interpretation. +""" +function finish!(stack, frame, pc=frame.pc[]) + while true + new_pc = _step_expr!(stack, frame, pc) + new_pc == nothing && break + pc = new_pc + end + frame.pc[] = pc +end + +""" + ret = finish_and_return!(stack, frame, pc=frame.pc[]) + +Run `frame` until execution terminates, and pass back the computed return value. +`stack` controls call evaluation; `stack = Compiled()` evaluates :call expressions +by normal dispatch, whereas a vector of `JuliaStackFrame`s will use recursive interpretation. +""" +function finish_and_return!(stack, frame, pc=frame.pc[]) + pc = finish!(stack, frame, pc) + node = pc_expr(frame, pc) + isexpr(node, :return) || error("unexpected node ", node) + return @eval_rhs(stack, frame, (node::Expr).args[1], pc) end -step_expr(frame) = _step_expr(frame, frame.pc) function is_call(node) isexpr(node, :call) || - (isexpr(node, :(=)) && isexpr(node.args[2], :call)) + (isexpr(node, :(=)) && (isexpr(node.args[2], :call))) end -function next_until!(f, frame, pc=frame.pc) - while (pc = _step_expr(frame, pc)) != nothing - f(pc_expr(frame, pc)) && return pc +""" + next_until!(predicate, stack, frame, pc=frame.pc[]) + +Step through statements of `frame` until the next statement satifies `predicate(stmt)`. +""" +function next_until!(f, stack, frame, pc=frame.pc[]) + while (pc = _step_expr!(stack, frame, pc)) != nothing + f(plain(pc_expr(frame, pc))) && (frame.pc[] = pc; return pc) end return nothing end -next_call!(frame, pc=frame.pc) = next_until!(node->is_call(node)||isexpr(node,:return), frame, pc) +next_call!(stack, frame, pc=frame.pc[]) = next_until!(node->is_call(node)||isexpr(node,:return), stack, frame, pc) function changed_line!(expr, line, fls) if length(fls) == 1 && isa(expr, LineNumberNode) @@ -202,67 +341,69 @@ function iswrappercall(expr) isexpr(expr, :call) && any(x->x==SlotNumber(1), expr.args) end -pc_expr(frame, pc) = frame.code.code[pc.next_stmt] -pc_expr(frame) = pc_expr(frame, frame.pc) +pc_expr(frame, pc) = frame.code.code.code[pc.next_stmt] +pc_expr(frame) = pc_expr(frame, frame.pc[]) function find_used(code::CodeInfo) used = BitSet() stmts = code.code for stmt in stmts - Core.Compiler.scan_ssa_use!(push!, used, stmt) + Core.Compiler.scan_ssa_use!(push!, used, plain(stmt)) end return used end -function maybe_next_call!(frame, pc) +function maybe_next_call!(stack, frame, pc) call_or_return(node) = is_call(node) || isexpr(node, :return) - call_or_return(pc_expr(frame, pc)) || - (pc = next_until!(call_or_return, frame, pc)) + call_or_return(plain(pc_expr(frame, pc))) || + (pc = next_until!(call_or_return, stack, frame, pc)) pc end -maybe_next_call!(frame) = maybe_next_call!(frame, frame.pc) +maybe_next_call!(stack, frame) = maybe_next_call!(stack, frame, frame.pc[]) -location(frame) = location(frame, frame.pc) -location(frame, pc) = frame.code.codelocs[pc.next_stmt] + frame.meth.line - 1 -function next_line!(frame, stack = nothing) +location(frame) = location(frame, frame.pc[]) +function location(frame, pc) + ln = frame.code.code.codelocs[pc.next_stmt] + return frame.code.scope isa Method ? ln + frame.code.scope.line - 1 : ln +end +function next_line!(stack, frame, dbstack = nothing) initial = location(frame) first = true - pc = frame.pc + pc = frame.pc[] while location(frame, pc) == initial # If this is a return node, interrupt execution. This is the same # special case as in `s`. - (!first && isexpr(pc_expr(frame, pc), :return)) && return pc + expr = plain(pc_expr(frame, pc)) + (!first && isexpr(expr, :return)) && return pc first = false # If this is a goto node, step it and reevaluate - if isgotonode(pc_expr(frame, pc)) - pc = _step_expr(frame, pc) + if isgotonode(expr) + pc = _step_expr!(stack, frame, pc) pc == nothing && return nothing - elseif stack !== nothing && iswrappercall(pc_expr(frame, pc)) + elseif dbstack !== nothing && iswrappercall(expr) # With splatting it can happen that we do something like ssa = tuple(#self#), _apply(ssa), which # confuses the logic here, just step into the first call that's not a builtin while true - stack[1] = JuliaStackFrame(frame, pc; wrapper = true) - call_expr = pc_expr(frame, pc) + dbstack[1] = JuliaStackFrame(JuliaFrameCode(frame.code; wrapper = true), frame, pc) + call_expr = plain(pc_expr(frame, pc)) isexpr(call_expr, :(=)) && (call_expr = call_expr.args[2]) - call_expr = Expr(:call, map(x->lookup_var_if_var(frame, x), call_expr.args)...) + call_expr = Expr(:call, map(x->@eval_rhs(true, frame, x, pc), call_expr.args)...) new_frame = enter_call_expr(call_expr) if new_frame !== nothing - pushfirst!(stack, new_frame) + pushfirst!(dbstack, new_frame) frame = new_frame - pc = frame.pc + pc = frame.pc[] break else - pc = _step_expr(frame, pc) + pc = _step_expr!(stack, frame, pc) pc == nothing && return nothing end end - elseif isa(pc_expr(frame, pc), LineNumberNode) - line != pc_expr(frame, pc).line && break - pc = _step_expr(frame, pc) else - pc = _step_expr(frame, pc) + pc = _step_expr!(stack, frame, pc) pc == nothing && return nothing end + frame.pc[] = pc end - maybe_next_call!(frame, pc) + maybe_next_call!(stack, frame, pc) end diff --git a/src/localmethtable.jl b/src/localmethtable.jl new file mode 100644 index 0000000..13f98ef --- /dev/null +++ b/src/localmethtable.jl @@ -0,0 +1,91 @@ +const max_methods = 4 # maximum number of MethodInstances tracked for a particular :call statement + +""" + framecode, lenv = get_call_framecode(fargs, parentframe::JuliaFrameCode, idx::Int) + +Return the framecode and environment for a call specified by `fargs = [f, args...]` (see [`prepare_args`](@ref)). +`parentframecode` is the caller, and `idx` is the program-counter index. +If possible, `framecode` will be looked up from the local method tables of `parentframe`. +""" +function get_call_framecode(fargs, parentframe::JuliaFrameCode, idx::Int) + nargs = length(fargs) # includes f as the first "argument" + # Determine whether we can look up the appropriate framecode in the local method table + if isassigned(parentframe.methodtables, idx) # if this is the first call, this may not yet be set + tme = tme1 = parentframe.methodtables[idx] + local tmeprev + depth = 1 + while true + # TODO: consider using world age bounds to handle cache invalidation + # Determine whether the argument types match the signature + sig = tme.sig.parameters::SimpleVector + if length(sig) == nargs + matches = true + for i = 1:nargs + if !isa(fargs[i], sig[i]) + matches = false + break + end + end + if matches + # Rearrange the list to place this method first + # (if we're in a loop, we'll likely match this one again on the next iteration) + if depth > 1 + parentframe.methodtables[idx] = tme + tmeprev.next = tme.next + tme.next = tme1 + end + # The framecode is stashed in the `inferred` field of the MethodInstance + mi = tme.func::MethodInstance + return mi.inferred::JuliaFrameCode, mi.sparam_vals + end + end + depth += 1 + tmeprev = tme + tme = tme.next + tme === nothing && break + tme = tme::TypeMapEntry + end + end + # We haven't yet encountered this argtype combination and need to look it up by dispatch + fargs[1] = f = to_function(fargs[1]) + if isa(f, Core.Builtin) + # See TODO in optimize! + return f(fargs[2:end]...), nothing # for code that has a direct call to a builtin + end + # HACK: don't recurse into inference. Inference sometimes returns SSAValue objects and this + # seems to confuse lookup_var. + if f === Base._return_type + return Base._return_type(fargs[2:end]...), nothing + end + framecode, args, env, argtypes = prepare_call(f, fargs) + # Store the results of the method lookup in the local method table + tme = ccall(:jl_new_struct_uninit, Any, (Any,), TypeMapEntry)::TypeMapEntry + tme.func = mi = ccall(:jl_new_struct_uninit, Any, (Any,), MethodInstance)::MethodInstance + tme.sig = mi.specTypes = argtypes + tme.isleafsig = true + tme.issimplesig = false + method = framecode.scope::Method + tme.va = method.isva + mi.def = method + mi.rettype = Any + mi.sparam_vals = env + mi.inferred = framecode # a slight abuse, but not insane + if isassigned(parentframe.methodtables, idx) + tme.next = parentframe.methodtables[idx] + # Drop the oldest tme, if necessary + tmetmp = tme.next + depth = 2 + while isdefined(tmetmp, :next) && tmetmp.next !== nothing + depth += 1 + tmetmp = tmetmp.next + depth >= max_methods && break + end + if depth >= max_methods + tmetmp.next = nothing + end + else + tme.next = nothing + end + parentframe.methodtables[idx] = tme + return framecode, env +end diff --git a/test/interpret.jl b/test/interpret.jl index cc6370a..6964558 100644 --- a/test/interpret.jl +++ b/test/interpret.jl @@ -75,3 +75,27 @@ function new_sym() end step_through(enter_call_expr(:($new_sym()))) + +function summer(A) + s = zero(eltype(A)) + for a in A + s += a + end + return s +end + +A = [0.12, -.99] +frame = ASTInterpreter2.enter_call(summer, A) +frame2 = ASTInterpreter2.enter_call(summer, A) +@test summer(A) == something(runframe(frame)) == something(runstack(frame2)) + +A = rand(1000) +@test @interpret(sum(A)) ≈ sum(A) # note: the compiler can leave things in registers to increase accuracy, doesn't happen with interpreted +fapply() = (Core.apply_type)(Base.NamedTuple, (), Tuple{}) +@test @interpret(fapply()) == fapply() +function fbc() + bc = Broadcast.broadcasted(CartesianIndex, 6, [1, 2, 3]) + copy(bc) +end +@test @interpret(fbc()) == fbc() +@test @interpret(repr("hi")) == repr("hi") # this tests kwargs and @generated functions diff --git a/test/misc.jl b/test/misc.jl index 67636e7..c1cc0de 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -15,3 +15,8 @@ state = dummy_state(stack) execute_command(state, state.stack[1], Val{:n}(), "n") execute_command(state, state.stack[1], Val{:finish}(), "finish") @test isempty(state.stack) + +@test runframe(ASTInterpreter2.enter_call(complicated_keyword_stuff, 1, 2)) == + runframe(@make_stack(complicated_keyword_stuff(1, 2))[1]) +@test runframe(ASTInterpreter2.enter_call(complicated_keyword_stuff, 1, 2; x=7, y=33)) == + runframe(@make_stack(complicated_keyword_stuff(1, 2; x=7, y=33))[1]) diff --git a/test/runtests.jl b/test/runtests.jl index 9dfd447..0742719 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,6 +4,7 @@ using REPL using DebuggerFramework +include("utils.jl") include("evaling.jl") include("stepping.jl") include("interpret.jl") diff --git a/test/stepping.jl b/test/stepping.jl index 2454885..1caecad 100644 --- a/test/stepping.jl +++ b/test/stepping.jl @@ -7,15 +7,6 @@ using DebuggerFramework: execute_command, dummy_state struct DummyState; end REPL.LineEdit.transition(s::DummyState, _) = nothing -# Steps through the whole expression using `s` -function step_through(frame) - state = DebuggerFramework.dummy_state([frame]) - while !isexpr(ASTInterpreter2.pc_expr(state.stack[end]), :return) - execute_command(state, state.stack[1], Val{:s}(), "s") - end - return ASTInterpreter2.lookup_var_if_var(state.stack[end], ASTInterpreter2.pc_expr(state.stack[end]).args[1]) -end - @assert step_through(ASTInterpreter2.enter_call_expr(:($(+)(1,2.5)))) == 3.5 @assert step_through(ASTInterpreter2.enter_call_expr(:($(sin)(1)))) == sin(1) @assert step_through(ASTInterpreter2.enter_call_expr(:($(gcd)(10,20)))) == gcd(10, 20) diff --git a/test/ui.jl b/test/ui.jl index f34fd83..4157582 100644 --- a/test/ui.jl +++ b/test/ui.jl @@ -1,7 +1,7 @@ using ASTInterpreter2, REPL # From base, but copied here to make sure we don't fail bacause base changed -function my_gcd(a::T, b::T) where T<:Union{Int64,UInt64,Int128,UInt128} +function my_gcd(a::T, b::T) where T<:Union{Int8,UInt8,Int16,UInt16,Int32,UInt32,Int64,UInt64,Int128,UInt128} a == 0 && return abs(b) b == 0 && return abs(a) za = trailing_zeros(a) @@ -18,11 +18,11 @@ function my_gcd(a::T, b::T) where T<:Union{Int64,UInt64,Int128,UInt128} end r = u << k # T(r) would throw InexactError; we want OverflowError instead - r > typemax(T) && throw1(a, b) + r > typemax(T) && throw(OverflowError("gcd($a, $b) overflows")) r % T end -if Sys.isunix() +if Sys.isunix() && VERSION >= v"1.1.0" using TerminalRegressionTests const thisdir = dirname(@__FILE__) @@ -33,7 +33,9 @@ if Sys.isunix() repl.interface = REPL.setup_interface(repl) repl.specialdisplay = REPL.REPLDisplay(repl) stack = ASTInterpreter2.@make_stack my_gcd(10, 20) - stack[1] = ASTInterpreter2.JuliaStackFrame(stack[1], stack[1].pc; fullpath=false) + stack[1] = ASTInterpreter2.JuliaStackFrame(stack[1], stack[1].pc[]; fullpath=false) DebuggerFramework.RunDebugger(stack, repl, emuterm) end +else + @warn "Skipping UI tests on non unix systems" end diff --git a/test/utils.jl b/test/utils.jl new file mode 100644 index 0000000..42cd15e --- /dev/null +++ b/test/utils.jl @@ -0,0 +1,22 @@ +using Base.Meta: isexpr +using ASTInterpreter2: JuliaStackFrame +using ASTInterpreter2: pc_expr, plain, evaluate_call!, finish_and_return!, @eval_rhs + +# Steps through the whole expression using `s` +function step_through(frame) + state = DebuggerFramework.dummy_state([frame]) + while !isexpr(plain(pc_expr(state.stack[end])), :return) + execute_command(state, state.stack[1], Val{:s}(), "s") + end + lastframe = state.stack[end] + return @eval_rhs(true, lastframe, plain(pc_expr(lastframe)).args[1], lastframe.pc[]) +end + +# Execute a frame using Julia's regular compiled-code dispatch for any :call expressions +runframe(frame, pc=frame.pc[]) = Some{Any}(finish_and_return!(Compiled(), frame, pc)) + +# Execute a frame using the interpreter for all :call expressions (except builtins & intrinsics) +function runstack(frame::JuliaStackFrame, pc=frame.pc[]) + stack = JuliaStackFrame[] + return Some{Any}(finish_and_return!(stack, frame, pc)) +end