Hanging on Windows on Julia 1.10 #393

wsmoses · 2024-03-13T22:57:44Z

using LLVM

function cpu_name()
    ccall(:jl_get_cpu_name, String, ())
end

function cpu_features()
    return ccall(:jl_get_cpu_features, String, ())
end

const modstr = """
; ModuleID = 'start'
source_filename = "start"
target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-w64-mingw32"

; Function Attrs: inaccessiblemem_or_argmemonly nofree
declare void @julia.safepoint(i64*) local_unnamed_addr #0

; Function Attrs: alwaysinline nofree
define [1 x double] @fwddiffejulia_identity_1243wrap(double %0, double %1) #1 !dbg !4 {
entry:
  %2 = call {}*** inttoptr (i64 140708411134816 to {}*** ()*)() #5
  %ptls_field3.i = getelementptr inbounds {}**, {}*** %2, i64 2
  %3 = bitcast {}*** %ptls_field3.i to i64***
  %ptls_load45.i = load i64**, i64*** %3, align 8, !tbaa !8, !alias.scope !12, !noalias !15
  %4 = getelementptr inbounds i64*, i64** %ptls_load45.i, i64 2
  %safepoint.i = load i64*, i64** %4, align 8, !tbaa !17, !alias.scope !19, !noalias !22
  fence syncscope("singlethread") seq_cst
  %5 = load volatile i64, i64* %safepoint.i, align 8, !dbg !24
  fence syncscope("singlethread") seq_cst
  %6 = insertvalue [1 x double] zeroinitializer, double %1, 0
  ret [1 x double] %6
}

; Function Attrs: inaccessiblemem_or_argmemonly
declare void @ijl_gc_queue_root({} addrspace(10)*) #2

; Function Attrs: allocsize(2)
declare noalias nonnull {} addrspace(10)* @ijl_gc_pool_alloc(i8*, i32, i32) #3

; Function Attrs: allocsize(1)
declare noalias nonnull {} addrspace(10)* @ijl_gc_big_alloc(i8*, i64) #4

; Function Attrs: allocsize(1)
declare noalias nonnull {} addrspace(10)* @ijl_gc_alloc_typed(i8*, i64, i8*) #4

attributes #0 = { inaccessiblemem_or_argmemonly nofree "enzyme_inactive" "enzymejl_world"="31504" }
attributes #1 = { alwaysinline nofree }
attributes #2 = { inaccessiblemem_or_argmemonly }
attributes #3 = { allocsize(2) }
attributes #4 = { allocsize(1) }
attributes #5 = { nounwind readnone }

!llvm.module.flags = !{!0, !1}
!llvm.dbg.cu = !{!2}

!0 = !{i32 2, !"Dwarf Version", i32 4}
!1 = !{i32 2, !"Debug Info Version", i32 3}
!2 = distinct !DICompileUnit(language: DW_LANG_Julia, file: !3, producer: "julia", isOptimized: true, runtimeVersion: 0, emissionKind: LineTablesOnly, nameTableKind: None)
!3 = !DIFile(filename: "julia", directory: ".")
!4 = distinct !DISubprogram(name: "fwddiffejulia_identity_1243wrap", linkageName: "fwddiffejulia_identity_1243wrap", scope: null, file: !5, type: !6, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !7)
!5 = !DIFile(filename: "operators.jl", directory: ".")
!6 = !DISubroutineType(types: !7)
!7 = !{}
!8 = !{!9, !9, i64 0}
!9 = !{!"jtbaa_gcframe", !10, i64 0}
!10 = !{!"jtbaa", !11, i64 0}
!11 = !{!"jtbaa"}
!12 = !{!13}
!13 = distinct !{!13, !14, !"primal"}
!14 = distinct !{!14, !" diff: %"}
!15 = !{!16}
!16 = distinct !{!16, !14, !"shadow_0"}
!17 = !{!18, !18, i64 0}
!18 = !{!"jtbaa_const", !10, i64 0}
!19 = !{!20}
!20 = distinct !{!20, !21, !"primal"}
!21 = distinct !{!21, !" diff: %ptls_load45"}
!22 = !{!23}
!23 = distinct !{!23, !21, !"shadow_0"}
!24 = !DILocation(line: 522, scope: !25, inlinedAt: !26)
!25 = distinct !DISubprogram(name: "identity", linkageName: "julia_identity_1243", scope: null, file: !5, line: 522, type: !6, scopeLine: 522, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !2, retainedNodes: !7)
!26 = distinct !DILocation(line: 0, scope: !4)
"""

function move_to_threadsafe(ir)
    LLVM.verify(ir) # try to catch broken modules

    # So 1. serialize the module
    buf = convert(MemoryBuffer, ir)

    # 2. deserialize and wrap by a ThreadSafeModule
    return ThreadSafeContext() do ctx
        mod = parse(LLVM.Module, buf)
        ThreadSafeModule(mod)
    end
end

optlevel = LLVM.API.LLVMCodeGenLevelNone

tempTM = LLVM.JITTargetMachine(LLVM.triple(), cpu_name(), cpu_features(); optlevel)
LLVM.asm_verbosity!(tempTM, true)

lljit = JuliaOJIT()

jd_main = JITDylib(lljit)

prefix = LLVM.get_prefix(lljit)
dg = LLVM.CreateDynamicLibrarySearchGeneratorForProcess(prefix)
LLVM.add!(jd_main, dg)


function absolute_symbol_materialization(name, ptr)
	address = LLVM.API.LLVMOrcJITTargetAddress(reinterpret(UInt, ptr))
	flags = LLVM.API.LLVMJITSymbolFlags(LLVM.API.LLVMJITSymbolGenericFlagsExported, 0)
	symbol = LLVM.API.LLVMJITEvaluatedSymbol(address, flags)
	gv = if LLVM.version() >= v"15"
		LLVM.API.LLVMOrcCSymbolMapPair(name, symbol)
	else
		LLVM.API.LLVMJITCSymbolMapPair(name, symbol)
	end
	return LLVM.absolute_symbols(Ref(gv))
end

function define_absolute_symbol(jd, name)
	ptr = LLVM.find_symbol(name)
	if ptr !== C_NULL
		LLVM.define(jd, absolute_symbol_materialization(name, ptr))
		return true
	end
	return false
end

if Sys.iswindows() && Int === Int64
	# TODO can we check isGNU?
	define_absolute_symbol(jd_main, mangle(lljit, "___chkstk_ms"))
end

es = ExecutionSession(lljit)
lctm = LLVM.LocalLazyCallThroughManager(triple(lljit), es)
ism = LLVM.LocalIndirectStubsManager(triple(lljit))

activate(LLVM.Context())

mod = parse(LLVM.Module, modstr)
adjoint_name = "fwddiffejulia_identity_1243wrap"

jd = LLVM.JITDylib(lljit)

tsm = move_to_threadsafe(mod)
LLVM.add!(lljit, jd, tsm)

LLVM.lookup(lljit, adjoint_name)

x/ref EnzymeAD/Enzyme.jl#1236

cc @vchuravy @gbaraldi @ArnoStrouwen @jeremiedb @leerosenthalj @Larbino1 @salbalkus @RGonTheNoble @ChrisRackauckas

maleadt · 2024-03-14T08:41:43Z

cc @vchuravy @gbaraldi @ArnoStrouwen @jeremiedb @leerosenthalj @Larbino1 @salbalkus @RGonTheNoble @ChrisRackauckas

Pinging everybody involved on what's going to be a low-level discussion does not seem useful to me.

Anyhow; I can reproduce on 1.11, and this seems to trigger a segfault under gdb, and fails an assertion when enabling them:

❯ jl +dev --project wip.jl |& c++filt
lookup
Assertion failed: catchjmp, file C:/Users/Tim/Julia/src/julia/src/debuginfo.cpp, line 314

[9196] signal 22: SIGABRT
in expression starting at C:\Users\Tim\Julia\pkg\LLVM\wip.jl:81
crt_sig_handler at C:/Users/Tim/Julia/src/julia/src\signals-win.c:95
raise at C:\Windows\System32\msvcrt.dll (unknown line)
abort at C:\Windows\System32\msvcrt.dll (unknown line)
assert at C:\Windows\System32\msvcrt.dll (unknown line)
registerJITObject at C:/Users/Tim/Julia/src/julia/src\debuginfo.cpp:314
jl_register_jit_object at C:/Users/Tim/Julia/src/julia/src\debuginfo.cpp:405
registerRTDyldJITObject at C:/Users/Tim/Julia/src/julia/src\jitlayers.cpp:1134
llvm::orc::RTDyldObjectLinkingLayer::onObjLoad(llvm::orc::MaterializationResponsibility&, llvm::object::ObjectFile const&, llvm::RuntimeDyld::MemoryManager&, llvm::RuntimeDyld::LoadedObjectInfo&, std::map<llvm::StringRef, llvm::JITEvaluatedSymbol, std::less<llvm::StringRef>, std::allocator<std::pair<llvm::StringRef const, llvm::JITEvaluatedSymbol> > >, std::set<llvm::StringRef, std::less<llvm::StringRef>, std::allocator<llvm::StringRef> >&) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::Error llvm::detail::UniqueFunctionBase<llvm::Error, llvm::object::ObjectFile const&, llvm::RuntimeDyld::LoadedObjectInfo&, std::map<llvm::StringRef, llvm::JITEvaluatedSymbol, std::less<llvm::StringRef>, std::allocator<std::pair<llvm::StringRef const, llvm::JITEvaluatedSymbol> > > >::CallImpl<llvm::orc::RTDyldObjectLinkingLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, std::unique_ptr<llvm::MemoryBuffer, std::default_delete<llvm::MemoryBuffer> >)::$_0>(void*, llvm::object::ObjectFile const&, llvm::RuntimeDyld::LoadedObjectInfo&, std::map<llvm::StringRef, llvm::JITEvaluatedSymbol, std::less<llvm::StringRef>, std::allocator<std::pair<llvm::StringRef const, llvm::JITEvaluatedSymbol> > >&) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::jitLinkForORC(llvm::object::OwningBinary<llvm::object::ObjectFile>, llvm::RuntimeDyld::MemoryManager&, llvm::JITSymbolResolver&, bool, llvm::unique_function<llvm::Error (llvm::object::ObjectFile const&, llvm::RuntimeDyld::LoadedObjectInfo&, std::map<llvm::StringRef, llvm::JITEvaluatedSymbol, std::less<llvm::StringRef>, std::allocator<std::pair<llvm::StringRef const, llvm::JITEvaluatedSymbol> > >)>, llvm::unique_function<void (llvm::object::OwningBinary<llvm::object::ObjectFile>, std::unique_ptr<llvm::RuntimeDyld::LoadedObjectInfo, std::default_delete<llvm::RuntimeDyld::LoadedObjectInfo> >, llvm::Error)>) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::RTDyldObjectLinkingLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, std::unique_ptr<llvm::MemoryBuffer, std::default_delete<llvm::MemoryBuffer> >) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
emit at C:/Users/Tim/Julia/src/julia/src\jitlayers.h:373
llvm::orc::IRCompileLayer::emit(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >, llvm::orc::ThreadSafeModule) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::BasicIRLayerMaterializationUnit::materialize(std::unique_ptr<llvm::orc::MaterializationResponsibility, std::default_delete<llvm::orc::MaterializationResponsibility> >) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::MaterializationTask::run() at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
void llvm::detail::UniqueFunctionBase<void, std::unique_ptr<llvm::orc::Task, std::default_delete<llvm::orc::Task> > >::CallImpl<void (*)(std::unique_ptr<llvm::orc::Task, std::default_delete<llvm::orc::Task> >)>(void*, std::unique_ptr<llvm::orc::Task, std::default_delete<llvm::orc::Task> >&) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::dispatchOutstandingMUs() at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::OL_completeLookup(std::unique_ptr<llvm::orc::InProgressLookupState, std::default_delete<llvm::orc::InProgressLookupState> >, std::shared_ptr<llvm::orc::AsynchronousSymbolQuery>, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::InProgressFullLookupState::complete(std::unique_ptr<llvm::orc::InProgressLookupState, std::default_delete<llvm::orc::InProgressLookupState> >) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::OL_applyQueryPhase1(std::unique_ptr<llvm::orc::InProgressLookupState, std::default_delete<llvm::orc::InProgressLookupState> >, llvm::Error) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::lookup(llvm::orc::LookupKind, std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolLookupSet, llvm::orc::SymbolState, llvm::unique_function<void (llvm::Expected<llvm::DenseMap<llvm::orc::SymbolStringPtr, llvm::JITEvaluatedSymbol, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void>, llvm::detail::DenseMapPair<llvm::orc::SymbolStringPtr, llvm::JITEvaluatedSymbol> > >)>, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::lookup(std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolLookupSet, llvm::orc::LookupKind, llvm::orc::SymbolState, std::function<void (llvm::DenseMap<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> >, llvm::DenseMapInfo<llvm::orc::JITDylib*, void>, llvm::detail::DenseMapPair<llvm::orc::JITDylib*, llvm::DenseSet<llvm::orc::SymbolStringPtr, llvm::DenseMapInfo<llvm::orc::SymbolStringPtr, void> > > > const&)>) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::lookup(std::vector<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags>, std::allocator<std::pair<llvm::orc::JITDylib*, llvm::orc::JITDylibLookupFlags> > > const&, llvm::orc::SymbolStringPtr, llvm::orc::SymbolState) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::lookup(llvm::ArrayRef<llvm::orc::JITDylib*>, llvm::orc::SymbolStringPtr, llvm::orc::SymbolState) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
llvm::orc::ExecutionSession::lookup(llvm::ArrayRef<llvm::orc::JITDylib*>, llvm::StringRef, llvm::orc::SymbolState) at C:\Users\Tim\Julia\src\julia\build\usr\bin\libLLVM-16jl.dll (unknown line)
findExternalJDSymbol at C:/Users/Tim/Julia/src/julia/src\jitlayers.cpp:1954
JLJITLookup_impl at C:/Users/Tim/Julia/src/julia/src\llvm_api.cpp:100
JLJITLookup at C:\Users\Tim\Julia\pkg\LLVM\lib\libLLVM_julia.jl:134
macro expansion at C:\Users\Tim\Julia\pkg\LLVM\src\executionengine\utils.jl:29 [inlined]
lookup at C:\Users\Tim\Julia\pkg\LLVM\src\orc.jl:410 [inlined]
lookup at C:\Users\Tim\Julia\pkg\LLVM\src\orc.jl:409

maleadt · 2024-03-14T09:18:14Z

MWE:

using LLVM

const modstr = """
    define void @foobar() {
    entry:
        ret void
    }"""

@dispose ts_ctx=ThreadSafeContext() lljit=JuliaOJIT() begin
    es = ExecutionSession(lljit)
    jd = JITDylib(lljit)

    lctm = LLVM.LocalLazyCallThroughManager(triple(lljit), es)
    ism = LLVM.LocalIndirectStubsManager(triple(lljit))
    try
        ctx = context(ts_ctx)
        context!(ctx) do
            mod = parse(LLVM.Module, modstr)
            tsm = ThreadSafeModule(mod)
            add!(lljit, jd, tsm)
            lookup(lljit, "foobar")
        end
    finally
        dispose(lctm)
        dispose(ism)
    end
end

Looks like the Julia JIT integration is fundamentally broken on Windows.

maleadt · 2024-03-14T09:26:36Z

We're missing the following inline assembly, that Julia's JIT expects: https://github.com/JuliaLang/julia/blob/f24364a4821f85410a27a236173c4e6bbb08e568/src/jitlayers.cpp#L2152-L2172

Since this is Julia-specific, I guess we could add this to add!(::JLJit, tsm).

wsmoses mentioned this issue Mar 13, 2024

Enzyme broken for Windows on Julia v1.10 EnzymeAD/Enzyme.jl#1236

Closed

maleadt mentioned this issue Mar 14, 2024

Fix Julia JIT integration on Windows x86_64 #394

Merged

maleadt closed this as completed in #394 Mar 14, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Hanging on Windows on Julia 1.10 #393

Hanging on Windows on Julia 1.10 #393

wsmoses commented Mar 13, 2024

maleadt commented Mar 14, 2024

maleadt commented Mar 14, 2024

maleadt commented Mar 14, 2024

Hanging on Windows on Julia 1.10 #393

Hanging on Windows on Julia 1.10 #393

Comments

wsmoses commented Mar 13, 2024

maleadt commented Mar 14, 2024

maleadt commented Mar 14, 2024

maleadt commented Mar 14, 2024