From 4170090765435726c9b81fa0236e256a4062b7a9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 25 Nov 2021 17:29:08 +0100 Subject: [PATCH] Multiversioning: support for aliases (from at-ccallable). (#37530) --- src/codegen-stubs.c | 2 +- src/llvm-multiversioning.cpp | 73 ++++++++++++++++++++++++++++++++---- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/src/codegen-stubs.c b/src/codegen-stubs.c index 163fa53443612..f328906cf2b4b 100644 --- a/src/codegen-stubs.c +++ b/src/codegen-stubs.c @@ -15,7 +15,7 @@ JL_DLLEXPORT void jl_dump_native_fallback(void *native_code, const char *sysimg_data, size_t sysimg_len) UNAVAILABLE JL_DLLEXPORT int32_t jl_get_llvm_gv_fallback(void *native_code, jl_value_t *p) UNAVAILABLE -JL_DLLEXPORT int jl_extern_c_fallback(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE +JL_DLLEXPORT void jl_extern_c_fallback(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE JL_DLLEXPORT jl_value_t *jl_dump_method_asm_fallback(jl_method_instance_t *linfo, size_t world, char raw_mc, char getwrapper, const char* asm_variant, const char *debuginfo, char binary) UNAVAILABLE JL_DLLEXPORT jl_value_t *jl_dump_function_ir_fallback(void *f, char strip_ir_metadata, char dump_module, const char *debuginfo) UNAVAILABLE diff --git a/src/llvm-multiversioning.cpp b/src/llvm-multiversioning.cpp index bc467ec12bff2..c937025fcdbca 100644 --- a/src/llvm-multiversioning.cpp +++ b/src/llvm-multiversioning.cpp @@ -272,13 +272,13 @@ struct CloneCtx { Constant *get_ptrdiff32(Constant *ptr, Constant *base) const; template Constant *emit_offset_table(const std::vector &vars, StringRef name) const; + void rewrite_alias(GlobalAlias *alias, Function* F); LLVMContext &ctx; Type *T_size; Type *T_int32; Type *T_void; PointerType *T_psize; - PointerType *T_pvoidfunc; MDNode *tbaa_const; MultiVersioning *pass; std::vector specs; @@ -295,6 +295,8 @@ struct CloneCtx { std::vector> gv_relocs{}; // Mapping from function id (i.e. 0-based index in `fvars`) to GVs to be initialized. std::map const_relocs; + // Functions that were referred to by a global alias, and might not have other uses. + std::set alias_relocs; bool has_veccall{false}; bool has_cloneall{false}; }; @@ -342,7 +344,6 @@ CloneCtx::CloneCtx(MultiVersioning *pass, Module &M) T_int32(Type::getInt32Ty(ctx)), T_void(Type::getVoidTy(ctx)), T_psize(PointerType::get(T_size, 0)), - T_pvoidfunc(FunctionType::get(T_void, false)->getPointerTo()), tbaa_const(tbaa_make_child("jtbaa_const", nullptr, true).first), pass(pass), specs(jl_get_llvm_clone_targets()), @@ -702,6 +703,54 @@ Constant *CloneCtx::rewrite_gv_init(const Stack& stack) return res; } +// replace an alias to a function with a trampoline and (uninitialized) global variable slot +void CloneCtx::rewrite_alias(GlobalAlias *alias, Function *F) +{ + assert(!is_vector(F->getFunctionType())); + + Function *trampoline = + Function::Create(F->getFunctionType(), alias->getLinkage(), "", &M); + trampoline->copyAttributesFrom(F); + trampoline->takeName(alias); + alias->eraseFromParent(); + + uint32_t id; + GlobalVariable *slot; + std::tie(id, slot) = get_reloc_slot(F); + for (auto &grp: groups) { + grp.relocs.insert(id); + for (auto &tgt: grp.clones) { + tgt.relocs.insert(id); + } + } + alias_relocs.insert(id); + + auto BB = BasicBlock::Create(ctx, "top", trampoline); + IRBuilder<> irbuilder(BB); + + auto ptr = irbuilder.CreateLoad(F->getType(), slot); + ptr->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_const); + ptr->setMetadata(llvm::LLVMContext::MD_invariant_load, MDNode::get(ctx, None)); + + std::vector Args; + for (auto &arg : trampoline->args()) + Args.push_back(&arg); + auto call = irbuilder.CreateCall(F->getFunctionType(), ptr, makeArrayRef(Args)); + if (F->isVarArg()) +#if (defined(_CPU_ARM_) || defined(_CPU_PPC_) || defined(_CPU_PPC64_)) + abort(); // musttail support is very bad on ARM, PPC, PPC64 (as of LLVM 3.9) +#else + call->setTailCallKind(CallInst::TCK_MustTail); +#endif + else + call->setTailCallKind(CallInst::TCK_Tail); + + if (F->getReturnType() == T_void) + irbuilder.CreateRetVoid(); + else + irbuilder.CreateRet(call); +} + void CloneCtx::fix_gv_uses() { auto single_pass = [&] (Function *orig_f) { @@ -712,8 +761,14 @@ void CloneCtx::fix_gv_uses() auto info = uses.get_info(); // We only support absolute pointer relocation. assert(info.samebits); - // And only for non-constant global variable initializers - auto val = cast(info.val); + GlobalVariable *val; + if (auto alias = dyn_cast(info.val)) { + rewrite_alias(alias, orig_f); + continue; + } + else { + val = cast(info.val); + } assert(info.use->getOperandNo() == 0); assert(!val->isConstant()); auto fid = get_func_id(orig_f); @@ -739,8 +794,8 @@ std::pair CloneCtx::get_reloc_slot(Function *F) auto id = get_func_id(F); auto &slot = const_relocs[id]; if (!slot) - slot = new GlobalVariable(M, T_pvoidfunc, false, GlobalVariable::InternalLinkage, - ConstantPointerNull::get(T_pvoidfunc), + slot = new GlobalVariable(M, F->getType(), false, GlobalVariable::InternalLinkage, + ConstantPointerNull::get(F->getType()), F->getName() + ".reloc_slot"); return std::make_pair(id, slot); } @@ -820,10 +875,9 @@ void CloneCtx::fix_inst_uses() uint32_t id; GlobalVariable *slot; std::tie(id, slot) = get_reloc_slot(orig_f); - Instruction *ptr = new LoadInst(T_pvoidfunc, slot, "", false, insert_before); + Instruction *ptr = new LoadInst(orig_f->getType(), slot, "", false, insert_before); ptr->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_const); ptr->setMetadata(llvm::LLVMContext::MD_invariant_load, MDNode::get(ctx, None)); - ptr = new BitCastInst(ptr, F->getType(), "", insert_before); use_i->setOperand(info.use->getOperandNo(), rewrite_inst_use(uses.get_stack(), ptr, insert_before)); @@ -951,6 +1005,9 @@ void CloneCtx::emit_metadata() values.push_back(id_v); values.push_back(get_ptrdiff32(it->second, gbase)); } + if (alias_relocs.find(id) != alias_relocs.end()) { + shared_relocs.insert(id); + } } values[0] = ConstantInt::get(T_int32, values.size() / 2); ArrayType *vars_type = ArrayType::get(T_int32, values.size());