A grab-bag of correctness fixes

src/JuliaInterpreter.jl

@@ -143,6 +143,15 @@ include("localmethtable.jl")
 include("interpret.jl")
 include("builtins.jl")
 
+function show_stackloc(io::IO, stack, frame, pc=frame.pc[])
+    indent = ""
+    for f in stack
+        println(io, indent, f.code.scope)
+        indent *= "  "
+    end
+    println(io, indent, frame.code.scope, ", pc = ", convert(Int, pc))
+end
+
 function moduleof(x)
     if isa(x, JuliaStackFrame)
         x = x.code.scope
@@ -234,6 +243,12 @@ function prepare_args(@nospecialize(f), allargs, kwargs)
     return f, allargs
 end
 
+function whichtt(tt)
+    m = ccall(:jl_gf_invoke_lookup, Any, (Any, UInt), tt, typemax(UInt))
+    m === nothing && return nothing
+    return m.func::Method
+end
+
 """
     framecode, frameargs, lenv, argtypes = prepare_call(f, allargs; enter_generated=false)
 
@@ -274,21 +289,16 @@ Tuple{typeof(mymethod),Array{Float64,1}}
 ```
 """
 function prepare_call(@nospecialize(f), allargs; enter_generated = false)
-    args = allargs[2:end]
-    argtypes = Tuple{map(_Typeof,args)...}
-    method = try
-        which(f, argtypes)
-    catch err
-        @show typeof(f)
-        println(f)
-        println(argtypes)
-        rethrow(err)
-    end
-    argtypes = Tuple{_Typeof(f), argtypes.parameters...}
+    argtypes = Tuple{map(_Typeof,allargs)...}
+    method = whichtt(argtypes)
+    if method === nothing
+        # Call it to generate the exact error
+        f(allargs[2:end]...)
+    end
     args = allargs
     sig = method.sig
     isa(method, TypeMapEntry) && (method = method.func)
-    if method ∈ compiled_methods
+    if method.module == Core.Compiler || method ∈ compiled_methods
         return Compiled()
     end
     # Get static parameters
@@ -445,15 +455,16 @@ function renumber_ssa!(stmts::Vector{Any}, ssalookup)
             stmts[i] = SSAValue(stmt.id)
         elseif isa(stmt, Expr)
             replace_ssa!(stmt, ssalookup)
-            if stmt.head == :gotoifnot && isa(stmt.args[2], Int)
-                stmt.args[2] = ssalookup[stmt.args[2]]
+            if (stmt.head == :gotoifnot || stmt.head == :enter) && isa(stmt.args[end], Int)
+                stmt.args[end] = ssalookup[stmt.args[end]]
             end
         end
     end
     return stmts
 end
 
 function lookup_global_refs!(ex::Expr)
+    isexpr(ex, :isdefined) && return nothing
     for (i, a) in enumerate(ex.args)
         if isa(a, GlobalRef)
             r = getfield(a.mod, a.name)
@@ -462,6 +473,7 @@ function lookup_global_refs!(ex::Expr)
             lookup_global_refs!(a)
         end
     end
+    return nothing
 end
 
 """
@@ -539,9 +551,9 @@ function prepare_locals(framecode, argvals::Vector{Any})
             exception_frames, last_reference = oldframe.exception_frames, oldframe.last_reference
             callargs = oldframe.callargs
             last_exception, pc = oldframe.last_exception, oldframe.pc
-            # for check_isdefined to work properly, we need locals and sparams to start out unassigned
-            resize!(resize!(locals, 0), length(code.slotflags))
+            resize!(locals, length(code.slotflags))
             resize!(ssavalues, ng)
+            # for check_isdefined to work properly, we need sparams to start out unassigned
             resize!(resize!(sparams, 0), length(meth.sparam_syms))
             empty!(exception_frames)
             empty!(last_reference)
@@ -571,6 +583,7 @@ function prepare_locals(framecode, argvals::Vector{Any})
     else
         code = framecode.code
         locals = Vector{Union{Nothing,Some{Any}}}(undef, length(code.slotflags))
+        fill!(locals, nothing)
         ssavalues = Vector{Any}(undef, length(code.code))
         sparams = Any[]
         exception_frames = Int[]

src/interpret.jl

@@ -10,7 +10,7 @@ lookup_var(frame, ref::GlobalRef) = getfield(ref.mod, ref.name)
 function lookup_var(frame, slot::SlotNumber)
     val = frame.locals[slot.id]
     val !== nothing && return val.value
-    error("slot ", slot, " not assigned")
+    error("slot ", slot, " with name ", frame.code.code.slotnames[slot.id], " not assigned")
 end
 
 function lookup_expr(frame, e::Expr)
@@ -106,9 +106,15 @@ end
 instantiate_type_in_env(arg, spsig, spvals) =
     ccall(:jl_instantiate_type_in_env, Any, (Any, Any, Ptr{Any}), arg, spsig, spvals)
 
-function resolvefc(@nospecialize expr)
-    (isa(expr, Symbol) || isa(expr, String) || isa(expr, QuoteNode)) && return expr
+function resolvefc(frame, @nospecialize expr)
+    if isa(expr, SlotNumber)
+        expr = lookup_var(frame, expr)
+    end
+    (isa(expr, Symbol) || isa(expr, String) || isa(expr, Ptr) || isa(expr, QuoteNode)) && return expr
     isa(expr, Tuple{Symbol,Symbol}) && return expr
+    isa(expr, Tuple{String,String}) && return expr
+    isa(expr, Tuple{Symbol,String}) && return expr
+    isa(expr, Tuple{String,Symbol}) && return expr
     if isexpr(expr, :call)
         a = expr.args[1]
         (isa(a, QuoteNode) && a.value == Core.tuple) || error("unexpected ccall to ", expr)
@@ -121,7 +127,7 @@ function collect_args(frame, call_expr; isfc=false)
     args = frame.callargs
     resize!(args, length(call_expr.args))
     mod = moduleof(frame)
-    args[1] = isfc ? resolvefc(call_expr.args[1]) : @lookup(mod, frame, call_expr.args[1])
+    args[1] = isfc ? resolvefc(frame, call_expr.args[1]) : @lookup(mod, frame, call_expr.args[1])
     for i = 2:length(args)
         args[i] = @lookup(mod, frame, call_expr.args[i])
     end
@@ -136,7 +142,7 @@ Evaluate a `:foreigncall` (from a `ccall`) statement `callexpr` in the context o
 """
 function evaluate_foreigncall!(stack, frame::JuliaStackFrame, call_expr::Expr, pc)
     args = collect_args(frame, call_expr; isfc=true)
-    for i = 1:length(args)
+    for i = 2:length(args)
         arg = args[i]
         args[i] = isa(arg, Symbol) ? QuoteNode(arg) : arg
     end
@@ -313,11 +319,11 @@ end
 
 function check_isdefined(frame, node)
     if isa(node, SlotNumber)
-        return isassigned(frame.locals, slot.id)
+        return frame.locals[node.id] !== nothing
     elseif isexpr(node, :static_parameter)
         return isassigned(frame.sparams, node.args[1]::Int)
     elseif isa(node, GlobalRef)
-        return isdefined(ref.mod, ref.name)
+        return isdefined(node.mod, node.name)
     elseif isa(node, Symbol)
         return isdefined(moduleof(frame), node)
     end

test/interpret.jl

@@ -116,4 +116,43 @@ end
 frame = JuliaInterpreter.prepare_toplevel(Main, ex)
 JuliaInterpreter.finish_and_return!(JuliaStackFrame[], frame, true)
 
-@test @interpret Base.Math.DoubleFloat64(-0.5707963267948967, 4.9789962508669555e-17).hi ≈ -0.5707963267948967
+@test @interpret Base.Math.DoubleFloat64(-0.5707963267948967, 4.9789962508669555e-17).hi ≈ -0.5707963267948967
+
+# ccall with cfunction
+fcfun(x::Int, y::Int) = 1
+ex = quote   # in lowered code, cf is a Symbol
+    cf = @eval @cfunction(fcfun, Int, (Int, Int))
+    ccall(cf, Int, (Int, Int), 1, 2)
+end
+frame = JuliaInterpreter.prepare_toplevel(Main, ex)
+@test JuliaInterpreter.finish_and_return!(JuliaStackFrame[], frame, true) == 1
+ex = quote
+    let   # in lowered code, cf is a SlotNumber
+        cf = @eval @cfunction(fcfun, Int, (Int, Int))
+        ccall(cf, Int, (Int, Int), 1, 2)
+    end
+end
+frame = JuliaInterpreter.prepare_toplevel(Main, ex)
+@test JuliaInterpreter.finish_and_return!(JuliaStackFrame[], frame, true) == 1
+
+# From Julia's test/ambiguous.jl. This tests whether we renumber :enter statements correctly.
+ambig(x, y) = 1
+ambig(x::Integer, y) = 2
+ambig(x, y::Integer) = 3
+ambig(x::Int, y::Int) = 4
+ambig(x::Number, y) = 5
+ex = quote
+    let
+        cf = @eval @cfunction(ambig, Int, (UInt8, Int))
+        @test_throws(MethodError, ccall(cf, Int, (UInt8, Int), 1, 2))
+    end
+end
+frame = JuliaInterpreter.prepare_toplevel(Main, ex)
+JuliaInterpreter.finish_and_return!(JuliaStackFrame[], frame, true)
+
+# Core.Compiler
+ex = quote
+    length(code_typed(fcfun, (Int, Int)))
+end
+frame = JuliaInterpreter.prepare_toplevel(Main, ex)
+@test JuliaInterpreter.finish_and_return!(JuliaStackFrame[], frame, true) == 1

test/juliatests.jl

@@ -40,37 +40,37 @@ end
         # empty!(JuliaInterpreter.genframedict)
         return JuliaInterpreter.finish_and_return!(stack, frame, true)
     end
-    function dotest!(failed, test)
+    function dotest!(test)
         println("Working on ", test, "...")
-        ex = read_and_parse(joinpath(testdir, test)*".jl")
+        fullpath = joinpath(testdir, test)*".jl"
+        ex = read_and_parse(fullpath)
+        # so `include` works properly, we have to set up the relative path
+        oldpath = current_task().storage[:SOURCE_PATH]
         if isexpr(ex, :error)
-            @warn "error parsing $test: $ex"
+            @error "error parsing $test: $ex"
         else
             try
+                current_task().storage[:SOURCE_PATH] = fullpath
                 lower_incrementally(runtest, JuliaTests, ex)
-                println("Succeeded on ", test)
-            catch err
-                @show test err
-                push!(failed, (test, err))
-                # rethrow(err)
+                # Core.eval(JuliaTests, ex)
+                println("Finished ", test)
+            finally
+                current_task().storage[:SOURCE_PATH] = oldpath
             end
         end
     end
     if isdir(testdir)
         tests, _ = choosetests()
         delayed = []
-        failed = []
         for test in tests
             if startswith(test, "compiler") || test == "subarray"
                 push!(delayed, test)
             else
-                dotest!(failed, test)
+                dotest!(test)
             end
         end
         for test in delayed
             dotest!(failed, test)
         end
-        @show failed
-        @test isempty(failed)
     end
 end

test/toplevel.jl

@@ -98,7 +98,7 @@ module Toplevel end
     @test @interpret(Toplevel.f3(1, :hi)) == 2
     @test @interpret(Toplevel.f3(UInt16(1), :hi)) == Symbol
     @test @interpret(Toplevel.f3(rand(2, 2), :hi, :there)) == 2
-    @test_throws ErrorException("no unique matching method found for the specified argument types") @interpret(Toplevel.f3([1.0], :hi, :there))
+    @test_throws MethodError @interpret(Toplevel.f3([1.0], :hi, :there))
     @test @interpret(Toplevel.f4(1, 1.0)) == 1
     @test @interpret(Toplevel.f4(1, 1)) == @interpret(Toplevel.f4(1)) == 2
     @test @interpret(Toplevel.f4(UInt(1), "hey", 2)) == 3
@@ -127,7 +127,7 @@ module Toplevel end
     @test @interpret(Toplevel.fouter(1)) === 2
     @test @interpret(Toplevel.feval1(1.0)) === 1
     @test @interpret(Toplevel.feval1(1.0f0)) === 1
-    @test_throws ErrorException("no unique matching method found for the specified argument types") @interpret(Toplevel.feval1(1))
+    @test_throws MethodError @interpret(Toplevel.feval1(1))
     @test @interpret(Toplevel.feval2(1.0, Int8(1))) == 2
     @test @interpret(length(s)) === nothing
     @test @interpret(size(s)) === nothing
@@ -152,3 +152,28 @@ module Toplevel end
    JuliaInterpreter.interpret!(stack, Toplevel, ex)
    @test Toplevel.Testing.JuliaStackFrame === JuliaStackFrame
 end
+
+module LowerAnon
+ret = Ref{Any}(nothing)
+end
+
+@testset "Anonymous functions" begin
+    ex1 = quote
+        f = x -> parse(Int16, x)
+        ret[] = map(f, AbstractString[])
+    end
+    ex2 = quote
+        ret[] = map(x->parse(Int16, x), AbstractString[])
+    end
+    stack = JuliaStackFrame[]
+    function runtest(frame)
+        empty!(stack)
+        return JuliaInterpreter.finish_and_return!(stack, frame, true)
+    end
+    lower_incrementally(runtest, LowerAnon, ex1)
+    @test isa(LowerAnon.ret[], Vector{Int16})
+    LowerAnon.ret[] = nothing
+    lower_incrementally(runtest, LowerAnon, ex2)
+    @test isa(LowerAnon.ret[], Vector{Int16})
+    LowerAnon.ret[] = nothing
+end

test/utils.jl

@@ -55,6 +55,16 @@ function lower_incrementally!(@nospecialize(f), docexprs, lex:: Expr, mod::Modul
             lower_incrementally!(f, docexprs, lex, mod, body)
         end
     else
+        # For map(x->x^2, a) we need to split out the anonymous function so that it
+        # gets defined prior to using it (essentially a world-age issue for inference)
+        if ex.head == :call || ex.head == :(=) && isa(ex.args[2], Expr) && ex.args[2].head == :call
+            fas = split_anonymous!(ex)
+            for fa in fas
+                lex1 = copy(lex)
+                push!(lex1.args, fa)
+                lower!(f, docexprs, mod, lex1)
+            end
+        end
         push!(lex.args, ex)
         lower!(f, docexprs, mod, lex)
         empty!(lex.args)
@@ -85,3 +95,19 @@ function lower!(@nospecialize(f), docexprs, mod::Module, ex::Expr)
     end
     return docexprs
 end
+
+split_anonymous!(ex) = split_anonymous!(Expr[], ex)
+function split_anonymous!(fs, ex)
+    for (i,a) in enumerate(ex.args)
+        if isa(a, Expr)
+            if a.head == :->
+                gs = gensym()
+                push!(fs, Expr(:(=), gs, a))
+                ex.args[i] = gs
+            else
+                split_anonymous!(fs, a)
+            end
+        end
+    end
+    return fs
+end