From d68e42f238602244d09e38f5d9e08e5b9c7a1465 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Wed, 6 Sep 2017 16:36:19 -0400 Subject: [PATCH] Fancier GC preserve intrinsics --- base/boot.jl | 2 -- base/pointer.jl | 21 +++++++++++ base/strings/string.jl | 5 +-- doc/src/devdocs/llvm.md | 21 +++++++++++ src/ast.c | 3 ++ src/ccall.cpp | 7 ---- src/codegen.cpp | 53 ++++++++++++++++++++++++--- src/interpreter.c | 5 +++ src/julia-syntax.scm | 10 ++++-- src/julia_internal.h | 1 + src/llvm-late-gc-lowering.cpp | 67 ++++++++++++++++++++++++++++++----- src/macroexpand.scm | 2 +- test/llvmpasses/gcroots.ll | 23 ++++++++++++ 13 files changed, 190 insertions(+), 30 deletions(-) diff --git a/base/boot.jl b/base/boot.jl index fe360e128d31a..dd5a425add0a9 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -438,6 +438,4 @@ show(@nospecialize a) = show(STDOUT, a) print(@nospecialize a...) = print(STDOUT, a...) println(@nospecialize a...) = println(STDOUT, a...) -gcuse(@nospecialize a) = ccall(:jl_gc_use, Void, (Any,), a) - ccall(:jl_set_istopmod, Void, (Any, Bool), Core, true) diff --git a/base/pointer.jl b/base/pointer.jl index 2e3e0e957d8e5..b2197d21db8c0 100644 --- a/base/pointer.jl +++ b/base/pointer.jl @@ -150,3 +150,24 @@ isless(x::Ptr, y::Ptr) = isless(UInt(x), UInt(y)) +(x::Ptr, y::Integer) = oftype(x, (UInt(x) + (y % UInt) % UInt)) -(x::Ptr, y::Integer) = oftype(x, (UInt(x) - (y % UInt) % UInt)) +(x::Integer, y::Ptr) = y + x + +""" +Temporarily protects an object from being garbage collected, even +if it would otherwise be unreferenced. + +The last argument is the expression to preserve objects during. +The previous arguments are the objects to preserve. +""" +macro gc_preserve(args...) + syms = args[1:end-1] + for x in syms + isa(x, Symbol) || error("Preserved variable must be a symbol") + end + s, r = gensym(), gensym() + esc(quote + $s = $(Expr(:gc_preserve_begin, syms...)) + $r = $(args[end]) + $(Expr(:gc_preserve_end, s)) + $r + end) +end diff --git a/base/strings/string.jl b/base/strings/string.jl index d81fe940fc8aa..efccebda556d0 100644 --- a/base/strings/string.jl +++ b/base/strings/string.jl @@ -73,10 +73,7 @@ codeunit(s::AbstractString, i::Integer) @boundscheck if (i < 1) | (i > sizeof(s)) throw(BoundsError(s,i)) end - ptr = pointer(s, i) - r = unsafe_load(ptr) - Core.gcuse(s) - r + @gc_preserve s unsafe_load(pointer(s, i)) end write(io::IO, s::String) = unsafe_write(io, pointer(s), reinterpret(UInt, sizeof(s))) diff --git a/doc/src/devdocs/llvm.md b/doc/src/devdocs/llvm.md index 63f0d2f3072ab..de176ea5afc0e 100644 --- a/doc/src/devdocs/llvm.md +++ b/doc/src/devdocs/llvm.md @@ -297,3 +297,24 @@ for the function. As a result, the external rooting must be arranged while the value is still tracked by the system. I.e. it is not valid to attempt to use the result of this operation to establish a global root - the optimizer may have already dropped the value. + +### Keeping values alive in the absence of uses + +In certain cases it is necessary to keep an object alive, even though there is +no compiler-visible use of said object. This may be case for low level code +that operates on the memory-representation of an object directly or code that +needs to interface with C code. In order to allow this, we provide the following +intrinsics at the LLVM level: +``` +token @llvm.julia.gc_preserve_begin(...) +void @llvm.julia.gc_preserve_end(token) +``` +(The `llvm.` in the name is required in order to be able to use the `token` +type). The semantics of these intrinsics are as follows: +At any safepoint that is dominated by a `gc_preserve_begin` call, but that is not +not dominated by a corresponding `gc_preserve_end` call (i.e. a call whose argument +is the token returned by a `gc_preserve_begin` call), the values passed as +arguments to that `gc_preserve_begin` will be kept live. Note that the +`gc_preserve_begin` still counts as a regular use of those values, so the +standard lifetime semantics will ensure that the values will be kept alive +before entering the preserve region. diff --git a/src/ast.c b/src/ast.c index 27378832f664b..1a75ab7a319f1 100644 --- a/src/ast.c +++ b/src/ast.c @@ -60,6 +60,7 @@ jl_sym_t *isdefined_sym; jl_sym_t *nospecialize_sym; jl_sym_t *macrocall_sym; jl_sym_t *hygienicscope_sym; jl_sym_t *escape_sym; +jl_sym_t *gc_preserve_begin_sym; jl_sym_t *gc_preserve_end_sym; static uint8_t flisp_system_image[] = { #include @@ -340,6 +341,8 @@ void jl_init_frontend(void) macrocall_sym = jl_symbol("macrocall"); escape_sym = jl_symbol("escape"); hygienicscope_sym = jl_symbol("hygienic-scope"); + gc_preserve_begin_sym = jl_symbol("gc_preserve_begin"); + gc_preserve_end_sym = jl_symbol("gc_preserve_end"); } JL_DLLEXPORT void jl_lisp_prompt(void) diff --git a/src/ccall.cpp b/src/ccall.cpp index 0554685f84148..4c7a8bc53bc75 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -1679,13 +1679,6 @@ static jl_cgval_t emit_ccall(jl_codectx_t &ctx, jl_value_t **args, size_t nargs) emit_signal_fence(ctx); return ghostValue(jl_void_type); } - else if (is_libjulia_func(jl_gc_use)) { - assert(lrt == T_void); - assert(!isVa && !llvmcall && nargt == 1); - ctx.builder.CreateCall(prepare_call(gc_use_func), {decay_derived(boxed(ctx, argv[0]))}); - JL_GC_POP(); - return ghostValue(jl_void_type); - } else if (_is_libjulia_func((uintptr_t)ptls_getter, "jl_get_ptls_states")) { assert(lrt == T_size); assert(!isVa && !llvmcall && nargt == 0); diff --git a/src/codegen.cpp b/src/codegen.cpp index f6589cfdd5355..5c2c1fb164192 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -356,7 +356,8 @@ static GlobalVariable *jlgetworld_global; // placeholder functions static Function *gcroot_flush_func; -static Function *gc_use_func; +static Function *gc_preserve_begin_func; +static Function *gc_preserve_end_func; static Function *except_enter_func; static Function *pointer_from_objref_func; @@ -3898,6 +3899,42 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr) else if (head == boundscheck_sym) { return mark_julia_const(bounds_check_enabled(ctx, jl_true) ? jl_true : jl_false); } + else if (head == gc_preserve_begin_sym) { + size_t nargs = jl_array_len(ex->args); + jl_cgval_t *argv = (jl_cgval_t*)alloca(sizeof(jl_cgval_t) * nargs); + for (size_t i = 0; i < nargs; ++i) { + argv[i] = emit_expr(ctx, args[i]); + } + size_t nargsboxed = 0; + Value **vals = (Value**)alloca(sizeof(Value *) * nargs); + for (size_t i = 0; i < nargs; ++i) { + if (!argv[i].isboxed) { + // This is intentionally not an error to allow writing + // generic code more easily. + continue; + } else if (argv[i].constant) { + continue; + } + vals[nargsboxed++] = argv[i].Vboxed; + } + Value *token = ctx.builder.CreateCall(prepare_call(gc_preserve_begin_func), + ArrayRef(vals, nargsboxed)); + jl_cgval_t tok(token, NULL, false, (jl_value_t*)jl_void_type, NULL); + tok.isimmutable = true; + return tok; + } + else if (head == gc_preserve_end_sym) { + // We only support ssa values as the argument. Everything else will + // fall back to the default behavior of preserving the argument value + // until the end of the scope, which is correct, but not optimal. + if (!jl_is_ssavalue(args[0])) { + return jl_cgval_t((jl_value_t*)jl_void_type); + } + jl_cgval_t token = emit_expr(ctx, args[0]); + assert(token.V->getType()->isTokenTy()); + ctx.builder.CreateCall(prepare_call(gc_preserve_end_func), {token.V}); + return jl_cgval_t((jl_value_t*)jl_void_type); + } else { if (!strcmp(jl_symbol_name(head), "$")) jl_error("syntax: prefix \"$\" in non-quoted expression"); @@ -6476,11 +6513,17 @@ static void init_julia_llvm_env(Module *m) "julia.gcroot_flush"); add_named_global(gcroot_flush_func, (void*)NULL, /*dllimport*/false); - gc_use_func = Function::Create(FunctionType::get(T_void, - ArrayRef(PointerType::get(T_jlvalue, AddressSpace::Derived)), false), + gc_preserve_begin_func = Function::Create(FunctionType::get(Type::getTokenTy(jl_LLVMContext), + ArrayRef(), true), Function::ExternalLinkage, - "julia.gc_use"); - add_named_global(gc_use_func, (void*)NULL, /*dllimport*/false); + "llvm.julia.gc_preserve_begin"); + add_named_global(gc_preserve_begin_func, (void*)NULL, /*dllimport*/false); + + gc_preserve_end_func = Function::Create(FunctionType::get(T_void, + ArrayRef(Type::getTokenTy(jl_LLVMContext)), false), + Function::ExternalLinkage, + "llvm.julia.gc_preserve_end"); + add_named_global(gc_preserve_end_func, (void*)NULL, /*dllimport*/false); pointer_from_objref_func = Function::Create(FunctionType::get(T_size, ArrayRef(PointerType::get(T_jlvalue, AddressSpace::Derived)), false), diff --git a/src/interpreter.c b/src/interpreter.c index 3892df26f90ac..e2d168f5ba05b 100644 --- a/src/interpreter.c +++ b/src/interpreter.c @@ -506,6 +506,11 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s) else if (ex->head == boundscheck_sym || ex->head == inbounds_sym || ex->head == fastmath_sym || ex->head == simdloop_sym || ex->head == meta_sym) { return jl_nothing; + } else if (ex->head == gc_preserve_begin_sym || ex->head == gc_preserve_end_sym) { + // The interpreter generally keeps values that were assigned in this scope + // rooted. If the interpreter learns to be more agressive here, we may + // want to explicitly root these values. + return jl_nothing; } jl_errorf("unsupported or misplaced expression %s", jl_symbol_name(ex->head)); abort(); diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index b78e091082434..d6f60f354832d 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -3084,7 +3084,8 @@ f(x) = yt(x) (memq (car e) '(quote top core line inert local local-def unnecessary meta inbounds boundscheck simdloop decl implicit-global global globalref outerref - const = null method call foreigncall ssavalue)))) + const = null method call foreigncall ssavalue + gc_preserve_begin gc_preserve_end)))) (lam:body lam)))) (unused (map cadr (filter (lambda (x) (memq (car x) '(method =))) leading)))) @@ -3802,8 +3803,13 @@ f(x) = yt(x) (if tail (emit-return '(null))) '(null)) + ((gc_preserve_begin) + (let ((s (make-ssavalue))) + (emit `(= ,s ,e)) + s)) + ;; other top level expressions and metadata - ((import importall using export line meta inbounds boundscheck simdloop) + ((import importall using export line meta inbounds boundscheck simdloop gc_preserve_end) (let ((have-ret? (and (pair? code) (pair? (car code)) (eq? (caar code) 'return)))) (cond ((eq? (car e) 'line) (set! current-loc e) diff --git a/src/julia_internal.h b/src/julia_internal.h index 94297906a56b3..9af96ffb689a3 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -999,6 +999,7 @@ extern jl_sym_t *propagate_inbounds_sym; extern jl_sym_t *isdefined_sym; extern jl_sym_t *nospecialize_sym; extern jl_sym_t *boundscheck_sym; +extern jl_sym_t *gc_preserve_begin_sym; extern jl_sym_t *gc_preserve_end_sym; void jl_register_fptrs(uint64_t sysimage_base, const char *base, const int32_t *offsets, jl_method_instance_t **linfos, size_t n); diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 4ef366c5b28f5..aeaaff0bf24b1 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -259,6 +259,10 @@ struct State { // of the value (but not the other way around). std::map LoadRefinements; + // GC preserves map. All safepoints dominated by the map key, but not any + // of its uses need to preserve the values listed in the map value. + std::map> GCPreserves; + // The assignment of numbers to safepoints. The indices in the map // are indices into the next three maps which store safepoint properties std::map SafepointNumbering; @@ -322,7 +326,8 @@ struct LateLowerGCFrame: public FunctionPass { MDNode *tbaa_tag; Function *ptls_getter; Function *gc_flush_func; - Function *gc_use_func; + Function *gc_preserve_begin_func; + Function *gc_preserve_end_func; Function *pointer_from_objref_func; Function *alloc_obj_func; Function *pool_alloc_func; @@ -754,17 +759,29 @@ State LateLowerGCFrame::LocalScan(Function &F) { if (CI->canReturnTwice()) { S.ReturnsTwice.push_back(CI); } - if (isa(CI)) { - // Intrinsics are never safepoints. - continue; - } if (auto callee = CI->getCalledFunction()) { + if (callee == gc_preserve_begin_func) { + std::vector args; + for (Use &U : CI->arg_operands()) { + Value *V = U; + int Num = Number(S, V); + if (Num >= 0) + args.push_back(Num); + } + S.GCPreserves[CI] = args; + continue; + } // Known functions emitted in codegen that are not safepoints - if (callee == pointer_from_objref_func || callee == gc_use_func || + if (callee == pointer_from_objref_func || callee == gc_preserve_begin_func || + callee == gc_preserve_end_func || callee->getName() == "memcmp") { continue; } } + if (isa(CI)) { + // Intrinsics are never safepoints. + continue; + } int SafepointNumber = NoteSafepoint(S, BBS, CI); BBS.HasSafepoint = true; BBS.TopmostSafepoint = SafepointNumber; @@ -912,6 +929,7 @@ JL_USED_FUNC static void dumpSafepointsForBBName(Function &F, State &S, const ch } void LateLowerGCFrame::ComputeLiveSets(Function &F, State &S) { + DominatorTree *DT = nullptr; // Iterate over all safe points. Add to live sets all those variables that // are now live across their parent block. for (auto it : S.SafepointNumbering) { @@ -935,6 +953,33 @@ void LateLowerGCFrame::ComputeLiveSets(Function &F, State &S) { if (RefinedPtr == -1 || HasBitSet(LS, RefinedPtr)) LS[Idx] = 0; } + // If the function has GC preserves, figure out whether we need to + // add in any extra live values. + if (!S.GCPreserves.empty()) { + if (!DT) { + DT = &getAnalysis().getDomTree(); + } + for (auto it2 : S.GCPreserves) { + if (!DT->dominates(it2.first, Safepoint)) + continue; + bool OutsideRange = false; + for (const User *U : it2.first->users()) { + // If this is dominated by an end, we don't need to add + // the values to our live set. + if (DT->dominates(cast(U), Safepoint)) { + OutsideRange = true; + break; + } + } + if (OutsideRange) + continue; + for (unsigned Num : it2.second) { + if (Num >= LS.size()) + LS.resize(Num + 1); + LS[Num] = 1; + } + } + } } // Compute the interference graph for (int i = 0; i <= S.MaxPtrNumber; ++i) { @@ -1153,8 +1198,8 @@ bool LateLowerGCFrame::CleanupIR(Function &F) { } CallingConv::ID CC = CI->getCallingConv(); auto callee = CI->getCalledValue(); - if ((gc_flush_func != nullptr && callee == gc_flush_func) || - (gc_use_func != nullptr && callee == gc_use_func)) { + if (callee && (callee == gc_flush_func || callee == gc_preserve_begin_func + || callee == gc_preserve_end_func)) { /* No replacement */ } else if (pointer_from_objref_func != nullptr && callee == pointer_from_objref_func) { auto *obj = CI->getOperand(0); @@ -1244,6 +1289,9 @@ bool LateLowerGCFrame::CleanupIR(Function &F) { NewCall->takeName(CI); CI->replaceAllUsesWith(NewCall); } + if (!CI->use_empty()) { + CI->replaceAllUsesWith(UndefValue::get(CI->getType())); + } it = CI->eraseFromParent(); ChangesMade = true; } @@ -1423,7 +1471,8 @@ static void addRetNoAlias(Function *F) bool LateLowerGCFrame::DefineFunctions(Module &M) { ptls_getter = M.getFunction("jl_get_ptls_states"); gc_flush_func = M.getFunction("julia.gcroot_flush"); - gc_use_func = M.getFunction("julia.gc_use"); + gc_preserve_begin_func = M.getFunction("llvm.julia.gc_preserve_begin"); + gc_preserve_end_func = M.getFunction("llvm.julia.gc_preserve_end"); pointer_from_objref_func = M.getFunction("julia.pointer_from_objref"); auto &ctx = M.getContext(); T_size = M.getDataLayout().getIntPtrType(ctx); diff --git a/src/macroexpand.scm b/src/macroexpand.scm index 6cd91ffdda26f..5a765b6807ff0 100644 --- a/src/macroexpand.scm +++ b/src/macroexpand.scm @@ -307,7 +307,7 @@ ,(resolve-expansion-vars-with-new-env (caddr arg) env m inarg)))) (else `(global ,(resolve-expansion-vars-with-new-env arg env m inarg)))))) - ((using import importall export meta line inbounds boundscheck simdloop) (map unescape e)) + ((using import importall export meta line inbounds boundscheck simdloop gc_preserve gc_preserve_end) (map unescape e)) ((macrocall) e) ; invalid syntax anyways, so just act like it's quoted. ((symboliclabel) e) ((symbolicgoto) e) diff --git a/test/llvmpasses/gcroots.ll b/test/llvmpasses/gcroots.ll index 7a92e64a0e434..2898fe62d13a1 100644 --- a/test/llvmpasses/gcroots.ll +++ b/test/llvmpasses/gcroots.ll @@ -223,3 +223,26 @@ top: ret void } +declare token @llvm.julia.gc_preserve_begin(...) +declare void @llvm.julia.gc_preserve_end(token) + +define void @gc_preserve(i64 %a) { +; CHECK-LABEL: @gc_preserve +; CHECK: %gcframe = alloca %jl_value_t addrspace(10)*, i32 4 +top: + %ptls = call %jl_value_t*** @jl_get_ptls_states() + %aboxed = call %jl_value_t addrspace(10)* @jl_box_int64(i64 signext %a) +; CHECK: store %jl_value_t addrspace(10)* %aboxed + call void @jl_safepoint() + %tok = call token (...) @llvm.julia.gc_preserve_begin(%jl_value_t addrspace(10)* %aboxed) + %aboxed2 = call %jl_value_t addrspace(10)* @jl_box_int64(i64 signext %a) +; CHECK: store %jl_value_t addrspace(10)* %aboxed2 + call void @jl_safepoint() + call void @llvm.julia.gc_preserve_end(token %tok) + %aboxed3 = call %jl_value_t addrspace(10)* @jl_box_int64(i64 signext %a) +; CHECK: store %jl_value_t addrspace(10)* %aboxed3 + call void @jl_safepoint() + call void @one_arg_boxed(%jl_value_t addrspace(10)* %aboxed2) + call void @one_arg_boxed(%jl_value_t addrspace(10)* %aboxed3) + ret void +}