diff --git a/src/codegen.cpp b/src/codegen.cpp index b99a9eff473fd..812a04313d9a7 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -7143,8 +7143,9 @@ static void allocate_gc_frame(jl_codectx_t &ctx, BasicBlock *b0, bool or_new=fal // allocate a placeholder gc instruction // this will require the runtime, but it gets deleted later if unused ctx.topalloca = ctx.builder.CreateCall(prepare_call(or_new ? jladoptthread_func : jlpgcstack_func)); - ctx.pgcstack = ctx.topalloca; - ctx.pgcstack->setName("pgcstack"); + ctx.topalloca->setName("pgcstack"); + if (ctx.pgcstack == nullptr) + ctx.pgcstack = ctx.topalloca; } static Value *get_current_task(jl_codectx_t &ctx) @@ -7291,7 +7292,6 @@ static void emit_specsig_to_specsig( ctx.builder.SetInsertPoint(b0); DebugLoc noDbg; ctx.builder.SetCurrentDebugLocation(noDbg); - allocate_gc_frame(ctx, b0); Function::arg_iterator AI = gf_thunk->arg_begin(); SmallVector myargs(nargs); if (cc == jl_returninfo_t::SRet || cc == jl_returninfo_t::Union) @@ -7299,8 +7299,10 @@ static void emit_specsig_to_specsig( if (return_roots) ++AI; if (JL_FEAT_TEST(ctx,gcstack_arg)) { + ctx.pgcstack = AI; ++AI; // gcstack_arg } + allocate_gc_frame(ctx, b0); for (size_t i = 0; i < nargs; i++) { if (i == 0 && is_for_opaque_closure) { // `jt` would be wrong here (it is the captures type), so is not used used for @@ -7407,6 +7409,10 @@ static void emit_specsig_to_specsig( break; } } + if (ctx.topalloca != ctx.pgcstack && ctx.topalloca->use_empty()) { + ctx.topalloca->eraseFromParent(); + ctx.topalloca = nullptr; + } } void emit_specsig_to_fptr1( @@ -8263,6 +8269,10 @@ static void gen_invoke_wrapper(jl_method_instance_t *lam, jl_value_t *abi, jl_va CreateTrap(ctx.builder, false); else ctx.builder.CreateRet(boxed(ctx, retval)); + if (ctx.topalloca != ctx.pgcstack && ctx.topalloca->use_empty()) { + ctx.topalloca->eraseFromParent(); + ctx.topalloca = nullptr; + } } static jl_returninfo_t get_specsig_function(jl_codegen_params_t ¶ms, Module *M, Value *fval, StringRef name, jl_value_t *sig, jl_value_t *jlrettype, bool is_opaque_closure, @@ -8919,7 +8929,53 @@ static jl_llvm_functions_t ctx.spvals_ptr = &*AI++; } } - // step 6. set up GC frame + // step 6. set up GC frame and special arguments + Function::arg_iterator AI = f->arg_begin(); + SmallVector attrs(f->arg_size()); // function declaration attributes + + if (has_sret) { + Argument *Arg = &*AI; + ++AI; + AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); + if (returninfo.cc == jl_returninfo_t::Union) { + param.addAttribute(Attribute::NonNull); + // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. + param.addDereferenceableAttr(returninfo.union_bytes); + param.addAlignmentAttr(returninfo.union_align); + } + else { + const DataLayout &DL = jl_Module->getDataLayout(); + Type *RT = Arg->getParamStructRetType(); + TypeSize sz = DL.getTypeAllocSize(RT); + Align al = DL.getPrefTypeAlign(RT); + if (al > MAX_ALIGN) + al = Align(MAX_ALIGN); + param.addAttribute(Attribute::NonNull); + // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. + param.addDereferenceableAttr(sz); + param.addAlignmentAttr(al); + } + attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes + } + if (returninfo.return_roots) { + Argument *Arg = &*AI; + ++AI; + AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); + param.addAttribute(Attribute::NonNull); + // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. + size_t size = returninfo.return_roots * sizeof(jl_value_t*); + param.addDereferenceableAttr(size); + param.addAlignmentAttr(Align(sizeof(jl_value_t*))); + attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes + } + if (specsig && JL_FEAT_TEST(ctx, gcstack_arg)) { + Argument *Arg = &*AI; + ctx.pgcstack = Arg; + ++AI; + AttrBuilder param(ctx.builder.getContext()); + attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); + } + allocate_gc_frame(ctx, b0); Value *last_age = NULL; Value *world_age_field = NULL; @@ -9062,9 +9118,6 @@ static jl_llvm_functions_t } // step 8. move args into local variables - Function::arg_iterator AI = f->arg_begin(); - SmallVector attrs(f->arg_size()); // function declaration attributes - auto get_specsig_arg = [&](jl_value_t *argType, Type *llvmArgType, bool isboxed) { if (type_is_ghost(llvmArgType)) { // this argument is not actually passed return ghostValue(ctx, argType); @@ -9097,47 +9150,6 @@ static jl_llvm_functions_t return theArg; }; - if (has_sret) { - Argument *Arg = &*AI; - ++AI; - AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); - if (returninfo.cc == jl_returninfo_t::Union) { - param.addAttribute(Attribute::NonNull); - // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. - param.addDereferenceableAttr(returninfo.union_bytes); - param.addAlignmentAttr(returninfo.union_align); - } - else { - const DataLayout &DL = jl_Module->getDataLayout(); - Type *RT = Arg->getParamStructRetType(); - TypeSize sz = DL.getTypeAllocSize(RT); - Align al = DL.getPrefTypeAlign(RT); - if (al > MAX_ALIGN) - al = Align(MAX_ALIGN); - param.addAttribute(Attribute::NonNull); - // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. - param.addDereferenceableAttr(sz); - param.addAlignmentAttr(al); - } - attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes - } - if (returninfo.return_roots) { - Argument *Arg = &*AI; - ++AI; - AttrBuilder param(ctx.builder.getContext(), f->getAttributes().getParamAttrs(Arg->getArgNo())); - param.addAttribute(Attribute::NonNull); - // The `dereferenceable` below does not imply `nonnull` for non addrspace(0) pointers. - size_t size = returninfo.return_roots * sizeof(jl_value_t*); - param.addDereferenceableAttr(size); - param.addAlignmentAttr(Align(sizeof(jl_value_t*))); - attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); // function declaration attributes - } - if (specsig && JL_FEAT_TEST(ctx, gcstack_arg)){ - Argument *Arg = &*AI; - ++AI; - AttrBuilder param(ctx.builder.getContext()); - attrs[Arg->getArgNo()] = AttributeSet::get(Arg->getContext(), param); - } for (i = 0; i < nreq && i < vinfoslen; i++) { jl_sym_t *s = slot_symbol(ctx, i); jl_varinfo_t &vi = ctx.slots[i]; @@ -10105,7 +10117,7 @@ static jl_llvm_functions_t } } - if (ctx.topalloca->use_empty()) { + if (ctx.topalloca != ctx.pgcstack && ctx.topalloca->use_empty()) { ctx.topalloca->eraseFromParent(); ctx.topalloca = nullptr; } diff --git a/src/llvm-final-gc-lowering.cpp b/src/llvm-final-gc-lowering.cpp index 76dcd944890ab..3ac2e61f25900 100644 --- a/src/llvm-final-gc-lowering.cpp +++ b/src/llvm-final-gc-lowering.cpp @@ -165,10 +165,6 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F) bool FinalLowerGC::runOnFunction(Function &F) { initAll(*F.getParent()); - if (!pgcstack_getter && !adoptthread_func) { - LLVM_DEBUG(dbgs() << "FINAL GC LOWERING: Skipping function " << F.getName() << "\n"); - return false; - } // Look for a call to 'julia.get_pgcstack'. pgcstack = getPGCstack(F); diff --git a/src/llvm-gc-interface-passes.h b/src/llvm-gc-interface-passes.h index 7b2a4bb033203..7a74000198a70 100644 --- a/src/llvm-gc-interface-passes.h +++ b/src/llvm-gc-interface-passes.h @@ -328,7 +328,7 @@ struct LateLowerGCFrame: private JuliaPassContext { bool runOnFunction(Function &F, bool *CFGModified = nullptr); private: - CallInst *pgcstack; + Value *pgcstack; Function *smallAllocFunc; void MaybeNoteDef(State &S, BBState &BBS, Value *Def, const ArrayRef &SafepointsSoFar, @@ -388,7 +388,7 @@ struct FinalLowerGC: private JuliaPassContext { Function *smallAllocFunc; Function *bigAllocFunc; Function *allocTypedFunc; - Instruction *pgcstack; + Value *pgcstack; Type *T_size; // Lowers a `julia.new_gc_frame` intrinsic. diff --git a/src/llvm-late-gc-lowering.cpp b/src/llvm-late-gc-lowering.cpp index 2b9b918c4bd53..cfca52a227c80 100644 --- a/src/llvm-late-gc-lowering.cpp +++ b/src/llvm-late-gc-lowering.cpp @@ -2330,7 +2330,10 @@ void LateLowerGCFrame::PlaceRootsAndUpdateCalls(ArrayRef Colors, int PreAss auto pushGcframe = CallInst::Create( getOrDeclare(jl_intrinsics::pushGCFrame), {gcframe, ConstantInt::get(T_int32, 0)}); - pushGcframe->insertAfter(pgcstack); + if (isa(pgcstack)) + pushGcframe->insertAfter(gcframe); + else + pushGcframe->insertAfter(cast(pgcstack)); // we don't run memsetopt after this, so run a basic approximation of it // that removes any redundant memset calls in the prologue since getGCFrameSlot already includes the null store @@ -2448,8 +2451,6 @@ bool LateLowerGCFrame::runOnFunction(Function &F, bool *CFGModified) { initAll(*F.getParent()); smallAllocFunc = getOrDeclare(jl_well_known::GCSmallAlloc); LLVM_DEBUG(dbgs() << "GC ROOT PLACEMENT: Processing function " << F.getName() << "\n"); - if (!pgcstack_getter && !adoptthread_func) - return CleanupIR(F, nullptr, CFGModified); pgcstack = getPGCstack(F); if (!pgcstack) diff --git a/src/llvm-pass-helpers.cpp b/src/llvm-pass-helpers.cpp index ca25251040fb2..38716cba521a1 100644 --- a/src/llvm-pass-helpers.cpp +++ b/src/llvm-pass-helpers.cpp @@ -72,19 +72,25 @@ void JuliaPassContext::initAll(Module &M) T_prjlvalue = JuliaType::get_prjlvalue_ty(ctx); } -llvm::CallInst *JuliaPassContext::getPGCstack(llvm::Function &F) const +llvm::Value *JuliaPassContext::getPGCstack(llvm::Function &F) const { - if (!pgcstack_getter && !adoptthread_func) - return nullptr; - for (auto &I : F.getEntryBlock()) { - if (CallInst *callInst = dyn_cast(&I)) { - Value *callee = callInst->getCalledOperand(); - if ((pgcstack_getter && callee == pgcstack_getter) || - (adoptthread_func && callee == adoptthread_func)) { - return callInst; + if (pgcstack_getter || adoptthread_func) { + for (auto &I : F.getEntryBlock()) { + if (CallInst *callInst = dyn_cast(&I)) { + Value *callee = callInst->getCalledOperand(); + if ((pgcstack_getter && callee == pgcstack_getter) || + (adoptthread_func && callee == adoptthread_func)) { + return callInst; + } } } } + if (F.getCallingConv() == CallingConv::Swift) { + for (auto &arg : F.args()) { + if (arg.hasSwiftSelfAttr()) + return &arg; + } + } return nullptr; } diff --git a/src/llvm-pass-helpers.h b/src/llvm-pass-helpers.h index d46f1f46634e6..d79470818c287 100644 --- a/src/llvm-pass-helpers.h +++ b/src/llvm-pass-helpers.h @@ -85,8 +85,9 @@ struct JuliaPassContext { // Gets a call to the `julia.get_pgcstack' intrinsic in the entry // point of the given function, if there exists such a call. + // Otherwise, gets a swiftself argument, if there exists such an argument. // Otherwise, `nullptr` is returned. - llvm::CallInst *getPGCstack(llvm::Function &F) const; + llvm::Value *getPGCstack(llvm::Function &F) const; // Gets the intrinsic or well-known function that conforms to // the given description if it exists in the module. If not, diff --git a/src/llvm-ptls.cpp b/src/llvm-ptls.cpp index e36136859517a..ebc7375af2ac7 100644 --- a/src/llvm-ptls.cpp +++ b/src/llvm-ptls.cpp @@ -317,13 +317,13 @@ bool LowerPTLS::run(bool *CFGModified) need_init = false; } - for (auto it = pgcstack_getter->user_begin(); it != pgcstack_getter->user_end();) { + for (auto it = pgcstack_getter->user_begin(); it != pgcstack_getter->user_end(); ) { auto call = cast(*it); ++it; auto f = call->getCaller(); Value *pgcstack = NULL; - for (Function::arg_iterator arg = f->arg_begin(); arg != f->arg_end();++arg) { - if (arg->hasSwiftSelfAttr()){ + for (Function::arg_iterator arg = f->arg_begin(); arg != f->arg_end(); ++arg) { + if (arg->hasSwiftSelfAttr()) { pgcstack = &*arg; break; }