Integrate interpreter frames into backtraces

base/deprecated.jl

@@ -95,8 +95,8 @@ function firstcaller(bt::Array{Ptr{Void},1}, funcsyms)
             found && @goto found
             found = lkup.func in funcsyms
             # look for constructor type name
-            if !found && !isnull(lkup.linfo)
-                li = get(lkup.linfo)
+            if !found && lkup.linfo != nothing
+                li = lkup.linfo
                 ft = ccall(:jl_first_argument_datatype, Any, (Any,), li.def.sig)
                 if isa(ft,DataType) && ft.name === Type.body.name
                     ft = unwrap_unionall(ft.parameters[1])

base/error.jl

@@ -60,12 +60,38 @@ Get a backtrace object for the current program point.
 """
 backtrace() = ccall(:jl_backtrace_from_here, Array{Ptr{Void},1}, (Int32,), false)
 
+struct InterpreterIP
+    code::CodeInfo
+    stmt::Csize_t
+end
+
 """
     catch_backtrace()
 
 Get the backtrace of the current exception, for use within `catch` blocks.
 """
-catch_backtrace() = ccall(:jl_get_backtrace, Array{Ptr{Void},1}, ())
+function catch_backtrace()
+    bt = Ref{Any}(nothing)
+    bt2 = Ref{Any}(nothing)
+    ccall(:jl_get_backtrace, Void, (Ref{Any}, Ref{Any}), bt, bt2)
+    ret = Array{Union{InterpreterIP,Ptr{Void}},1}()
+    i, j = 1, 1
+    while i <= length(bt[])
+        ip = bt[][i]::Ptr{Void}
+        if ip == Ptr{Void}(-1%UInt)
+            # The next one is really a CodeInfo
+            push!(ret, InterpreterIP(
+                bt2[][j],
+                bt[][i+2]))
+            j += 1
+            i += 3
+        else
+            push!(ret, Ptr{Void}(ip))
+            i += 1
+        end
+    end
+    ret
+end
 
 ## keyword arg lowering generates calls to this ##
 function kwerr(kw, args::Vararg{Any,N}) where {N}

base/stacktraces.jl

@@ -20,7 +20,7 @@ Stack information representing execution context, with the following fields:
 
   The name of the function containing the execution context.
 
-- `linfo::Nullable{Core.MethodInstance}`
+- `linfo::Union{Core.MethodInstance, CodeInfo, Void}`
 
   The MethodInstance containing the execution context (if it could be found).
 
@@ -52,8 +52,8 @@ struct StackFrame # this type should be kept platform-agnostic so that profiles
     file::Symbol
     "the line number in the file containing the execution context"
     line::Int
-    "the MethodInstance containing the execution context (if it could be found)"
-    linfo::Nullable{Core.MethodInstance}
+    "the MethodInstance or CodeInfo containing the execution context (if it could be found)"
+    linfo::Union{Core.MethodInstance, CodeInfo, Void}
     "true if the code is from C"
     from_c::Bool
     "true if the code is from an inlined frame"
@@ -63,7 +63,7 @@ struct StackFrame # this type should be kept platform-agnostic so that profiles
 end
 
 StackFrame(func, file, line) = StackFrame(Symbol(func), Symbol(file), line,
-                                          Nullable{Core.MethodInstance}(), false, false, 0)
+                                          nothing, false, false, 0)
 
 """
     StackTrace
@@ -74,7 +74,7 @@ An alias for `Vector{StackFrame}` provided for convenience; returned by calls to
 const StackTrace = Vector{StackFrame}
 
 const empty_sym = Symbol("")
-const UNKNOWN = StackFrame(empty_sym, empty_sym, -1, Nullable{Core.MethodInstance}(), true, false, 0) # === lookup(C_NULL)
+const UNKNOWN = StackFrame(empty_sym, empty_sym, -1, nothing, true, false, 0) # === lookup(C_NULL)
 
 
 #=
@@ -114,7 +114,7 @@ function deserialize(s::AbstractSerializer, ::Type{StackFrame})
     from_c = read(s.io, Bool)
     inlined = read(s.io, Bool)
     pointer = read(s.io, UInt64)
-    return StackFrame(func, file, line, Nullable{Core.MethodInstance}(), from_c, inlined, pointer)
+    return StackFrame(func, file, line, nothing, from_c, inlined, pointer)
 end
 
 
@@ -127,19 +127,59 @@ inlined at that point, innermost function first.
 """
 function lookup(pointer::Ptr{Void})
     infos = ccall(:jl_lookup_code_address, Any, (Ptr{Void}, Cint), pointer - 1, false)
-    isempty(infos) && return [StackFrame(empty_sym, empty_sym, -1, Nullable{Core.MethodInstance}(), true, false, convert(UInt64, pointer))]
+    isempty(infos) && return [StackFrame(empty_sym, empty_sym, -1, nothing, true, false, convert(UInt64, pointer))]
     res = Vector{StackFrame}(length(infos))
     for i in 1:length(infos)
         info = infos[i]
         @assert(length(info) == 7)
-        li = info[4] === nothing ? Nullable{Core.MethodInstance}() : Nullable{Core.MethodInstance}(info[4])
-        res[i] = StackFrame(info[1], info[2], info[3], li, info[5], info[6], info[7])
+        res[i] = StackFrame(info[1], info[2], info[3], info[4], info[5], info[6], info[7])
     end
     return res
 end
 
 lookup(pointer::UInt) = lookup(convert(Ptr{Void}, pointer))
 
+using Base.Meta
+is_loc_meta(expr, kind) = isexpr(expr, :meta) && length(expr.args) >= 1 && expr.args[1] === kind
+function lookup(ip::Base.InterpreterIP)
+    i = ip.stmt
+    foundline = false
+    func = empty_sym
+    file = empty_sym
+    line = 0
+    while i >= 1
+        expr = ip.code.code[i]
+        if !foundline && isa(expr, LineNumberNode)
+            line = expr.line
+            file = expr.file
+            foundline = true
+        elseif foundline && is_loc_meta(expr, :push_loc)
+            file = expr.args[2]
+            if length(expr.args) >= 3
+                func = expr.args[3]
+            else
+                # Note: This is not quite correct. See issue #23971
+                func = Symbol("macro expansion")
+            end
+            scopes = lookup(Base.InterpreterIP(ip.code, i-1))
+            unshift!(scopes, StackFrame(
+                func, file, line, nothing, false, true, 0
+            ))
+            return scopes
+        elseif is_loc_meta(expr, :pop_loc)
+            npops = 1
+            while npops >= 1
+                i -= 1
+                expr = ip.code.code[i]
+                is_loc_meta(expr, :pop_loc) && (npops += 1)
+                is_loc_meta(expr, :push_loc) && (npops -= 1)
+            end
+        end
+        i -= 1
+    end
+    return [StackFrame(func, file, line, ip.code, false, false, 0)]
+end
+
 # allow lookup on already-looked-up data for easier handling of pre-processed frames
 lookup(s::StackFrame) = StackFrame[s]
 lookup(s::Tuple{StackFrame,Int}) = StackFrame[s[1]]
@@ -151,7 +191,7 @@ Returns a stack trace in the form of a vector of `StackFrame`s. (By default stac
 doesn't return C functions, but this can be enabled.) When called without specifying a
 trace, `stacktrace` first calls `backtrace`.
 """
-function stacktrace(trace::Vector{Ptr{Void}}, c_funcs::Bool=false)
+function stacktrace(trace::Vector{<:Union{Base.InterpreterIP,Ptr{Void}}}, c_funcs::Bool=false)
     stack = vcat(StackTrace(), map(lookup, trace)...)::StackTrace
 
     # Remove frames that come from C calls.
@@ -209,19 +249,20 @@ function remove_frames!(stack::StackTrace, m::Module)
 end
 
 function show_spec_linfo(io::IO, frame::StackFrame)
-    if isnull(frame.linfo)
+    if frame.linfo == nothing
         if frame.func === empty_sym
             @printf(io, "ip:%#x", frame.pointer)
         else
             print_with_color(Base.have_color && get(io, :backtrace, false) ? Base.stackframe_function_color() : :nothing, io, string(frame.func))
         end
-    else
-        linfo = get(frame.linfo)
-        if isa(linfo.def, Method)
-            Base.show_tuple_as_call(io, linfo.def.name, linfo.specTypes)
+    elseif frame.linfo isa Core.MethodInstance
+        if isa(frame.linfo.def, Method)
+            Base.show_tuple_as_call(io, frame.linfo.def.name, frame.linfo.specTypes)
         else
-            Base.show(io, linfo)
+            Base.show(io, frame.linfo)
         end
+    elseif frame.linfo isa CodeInfo
+        print(io, "In toplevel scope")
     end
 end
 
@@ -253,8 +294,8 @@ function from(frame::StackFrame, m::Module)
     finfo = frame.linfo
     result = false
 
-    if !isnull(finfo)
-        frame_m = get(finfo).def
+    if finfo isa Core.MethodInstance
+        frame_m = finfo.def
         isa(frame_m, Method) && (frame_m = frame_m.module)
         result = module_name(frame_m) === module_name(m)
     end

base/task.jl

@@ -7,8 +7,8 @@ struct CapturedException <: Exception
     ex::Any
     processed_bt::Vector{Any}
 
-    function CapturedException(ex, bt_raw::Vector{Ptr{Void}})
-        # bt_raw MUST be an Array of code pointers than can be processed by jl_lookup_code_address
+    function CapturedException(ex, bt_raw::Vector)
+        # bt_raw MUST be a vector that can be processed by StackTraces.stacktrace
         # Typically the result of a catch_backtrace()
 
         # Process bt_raw so that it can be safely serialized

base/util.jl

@@ -350,8 +350,8 @@ const log_warn_to = Dict{Tuple{Union{Module,Void},Union{Symbol,Void}},IO}()
 const log_error_to = Dict{Tuple{Union{Module,Void},Union{Symbol,Void}},IO}()
 
 function _redirect(io::IO, log_to::Dict, sf::StackTraces.StackFrame)
-    isnull(sf.linfo) && return io
-    mod = get(sf.linfo).def
+    (sf.linfo isa Core.MethodInstance) || return io
+    mod = sf.linfo.def
     isa(mod, Method) && (mod = mod.module)
     fun = sf.func
     if haskey(log_to, (mod,fun))
@@ -374,10 +374,10 @@ function _redirect(io::IO, log_to::Dict, fun::Symbol)
         stack::Vector{StackFrame} = StackTraces.lookup(trace)
         filter!(frame -> !frame.from_c, stack)
         for frame in stack
-            isnull(frame.linfo) && continue
+            (frame.linfo isa Core.MethodInstance) || continue
             sf = frame
             break_next_frame && (@goto skip)
-            mod = get(frame.linfo).def
+            mod = frame.linfo.def
             isa(mod, Method) && (mod = mod.module)
             mod === Base || continue
             sff = string(frame.func)

src/Makefile

@@ -184,6 +184,7 @@ $(BUILDDIR)/ast.o $(BUILDDIR)/ast.dbg.obj: $(BUILDDIR)/julia_flisp.boot.inc $(SR
 $(BUILDDIR)/codegen.o $(BUILDDIR)/codegen.dbg.obj: $(addprefix $(SRCDIR)/,\
 	intrinsics.cpp jitlayers.h intrinsics.h debuginfo.h codegen_shared.h cgutils.cpp ccall.cpp abi_*.cpp processor.h)
 $(BUILDDIR)/processor.o $(BUILDDIR)/processor.dbg.obj: $(addprefix $(SRCDIR)/,processor_*.cpp processor.h features_*.h)
+$(BUILDDIR)/interpreter.o $(BUILDDIR)/interpreter.dbg.obj: $(SRCDIR)/interpreter-stacktrace.c
 $(BUILDDIR)/anticodegen.o $(BUILDDIR)/anticodegen.dbg.obj: $(SRCDIR)/intrinsics.h
 $(BUILDDIR)/debuginfo.o $(BUILDDIR)/debuginfo.dbg.obj: \
 	$(addprefix $(SRCDIR)/,debuginfo.h processor.h)

src/gc.c

@@ -194,17 +194,22 @@ static void jl_gc_run_finalizers_in_list(jl_ptls_t ptls, arraylist_t *list)
     // from jl_apply_with_saved_exception_state; to hoist state saving out of the loop
     jl_value_t *exc = ptls->exception_in_transit;
     jl_array_t *bt = NULL;
-    JL_GC_PUSH2(&exc, &bt);
-    if (ptls->bt_size > 0)
-        bt = (jl_array_t*)jl_get_backtrace();
+    jl_array_t *bt2 = NULL;
+    JL_GC_PUSH3(&exc, &bt, &bt2);
+    if (ptls->bt_size > 0) {
+        jl_get_backtrace(&bt, &bt2);
+        ptls->bt_size = 0;
+    }
     for (size_t i = 2;i < len;i += 2)
         run_finalizer(ptls, items[i], items[i + 1]);
     ptls->exception_in_transit = exc;
     if (bt != NULL) {
+        // This is sufficient because bt2 roots the managed values
+        memcpy(ptls->bt_data, bt->data, jl_array_len(bt) * sizeof(void*));
         ptls->bt_size = jl_array_len(bt);
-        memcpy(ptls->bt_data, bt->data, ptls->bt_size * sizeof(void*));
     }
     JL_GC_POP();
+    // matches the jl_gc_push_arraylist above
     JL_GC_POP();
 }
 
@@ -2434,6 +2439,18 @@ static void jl_gc_queue_remset(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, j
     ptls2->heap.rem_bindings.len = n_bnd_refyoung;
 }
 
+static void jl_gc_queue_bt_buf(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, jl_ptls_t ptls2)
+{
+    size_t n = 0;
+    while (n+2 < ptls2->bt_size) {
+        if (ptls2->bt_data[n] == (uintptr_t)-1) {
+            gc_mark_queue_obj(gc_cache, sp, (jl_value_t*)ptls2->bt_data[n+1]);
+            n += 2;
+        }
+        n++;
+    }
+}
+
 // Only one thread should be running in this function
 static int _jl_gc_collect(jl_ptls_t ptls, int full)
 {
@@ -2454,6 +2471,8 @@ static int _jl_gc_collect(jl_ptls_t ptls, int full)
         jl_gc_queue_remset(gc_cache, &sp, ptls2);
         // 2.2. mark every thread local root
         jl_gc_queue_thread_local(gc_cache, &sp, ptls2);
+        // 2.3. mark any managed objects in the backtrace buffer
+        jl_gc_queue_bt_buf(gc_cache, &sp, ptls2);
     }
 
     // 3. walk roots