diff --git a/src/UnsafeAtomics.jl b/src/UnsafeAtomics.jl index a83b8d2..988a784 100644 --- a/src/UnsafeAtomics.jl +++ b/src/UnsafeAtomics.jl @@ -1,6 +1,7 @@ baremodule UnsafeAtomics abstract type Ordering end +abstract type SyncScope end function load end function store! end @@ -26,10 +27,11 @@ module Internal using Base.Sys: WORD_SIZE using Base: bitcast, llvmcall -using ..UnsafeAtomics: UnsafeAtomics, Ordering, right +using ..UnsafeAtomics: UnsafeAtomics, Ordering, SyncScope, right include("utils.jl") include("orderings.jl") +include("syncscopes.jl") include("core.jl") end # module Internal @@ -45,4 +47,8 @@ const seq_cst = Internal.seq_cst const acquire_release = acq_rel const sequentially_consistent = seq_cst +# SyncScope +const none = Internal.none +const singlethread = Internal.singlethread + end # baremodule UnsafeAtomics diff --git a/src/core.jl b/src/core.jl index aff57f8..3210d8d 100644 --- a/src/core.jl +++ b/src/core.jl @@ -4,6 +4,12 @@ @inline UnsafeAtomics.modify!(ptr, op, x) = UnsafeAtomics.modify!(ptr, op, x, seq_cst) @inline UnsafeAtomics.fence() = UnsafeAtomics.fence(seq_cst) +@inline UnsafeAtomics.load(x, ord) = UnsafeAtomics.load(x, ord, none) +@inline UnsafeAtomics.store!(x, v, ord) = UnsafeAtomics.store!(x, v, ord, none) +@inline UnsafeAtomics.cas!(x, cmp, new, ord) = UnsafeAtomics.cas!(x, cmp, new, ord, ord, none) +@inline UnsafeAtomics.modify!(ptr, op, x, ord) = UnsafeAtomics.modify!(ptr, op, x, ord, none) +@inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord, none) + #! format: off # https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl#L23-L30 if Sys.ARCH == :i686 || startswith(string(Sys.ARCH), "arm") || @@ -45,8 +51,9 @@ const OP_RMW_TABLE = [ for (op, rmwop) in OP_RMW_TABLE fn = Symbol(rmwop, "!") @eval @inline UnsafeAtomics.$fn(x, v) = UnsafeAtomics.$fn(x, v, seq_cst) - @eval @inline UnsafeAtomics.$fn(ptr, x, ord) = - first(UnsafeAtomics.modify!(ptr, $op, x, ord)) + @eval @inline UnsafeAtomics.$fn(x, v, ord) = UnsafeAtomics.$fn(x, v, ord, none) + @eval @inline UnsafeAtomics.$fn(ptr, x, ord, scope) = + first(UnsafeAtomics.modify!(ptr, $op, x, ord, scope)) end const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref) @@ -67,47 +74,51 @@ for typ in (inttypes..., floattypes...) for ord in orderings ord in (release, acq_rel) && continue - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) - return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord)) - end - else - @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ}}, - x, - ) + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync))) + return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = load atomic $rt %ptr $ord, align $(sizeof(typ)) + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ}}, + x, + ) + end end end end for ord in orderings ord in (acquire, acq_rel) && continue - - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) - Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord)) - return nothing - end - else - @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord))) - return llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) - ret void - """), - Cvoid, - Tuple{Ptr{$typ},$typ}, - x, - v, - ) + + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync))) + Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord)) + return nothing + end + else + @eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ)) + ret void + """), + Cvoid, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + end end end end @@ -117,54 +128,58 @@ for typ in (inttypes..., floattypes...) typ <: AbstractFloat && break - if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.cas!( - x::Ptr{$typ}, - cmp::$typ, - new::$typ, - ::$(typeof(success_ordering)), - ::$(typeof(failure_ordering)), - ) - return Core.Intrinsics.atomic_pointerreplace( - x, - cmp, - new, - base_ordering($success_ordering), - base_ordering($failure_ordering) + for sync in syncscopes + if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ::$(typeof(sync)), ) - end - else - @eval function UnsafeAtomics.cas!( - x::Ptr{$typ}, - cmp::$typ, - new::$typ, - ::$(typeof(success_ordering)), - ::$(typeof(failure_ordering)), - ) - success = Ref{Int8}() - GC.@preserve success begin - old = llvmcall( - $( - """ - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering - %rv = extractvalue { $lt, i1 } %rs, 0 - %s1 = extractvalue { $lt, i1 } %rs, 1 - %s8 = zext i1 %s1 to i8 - %sptr = inttoptr i$WORD_SIZE %3 to i8* - store i8 %s8, i8* %sptr - ret $lt %rv - """ - ), - $typ, - Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + return Core.Intrinsics.atomic_pointerreplace( x, cmp, new, - Ptr{Int8}(pointer_from_objref(success)), + base_ordering($success_ordering), + base_ordering($failure_ordering) ) end - return (old = old, success = !iszero(success[])) + else + @eval function UnsafeAtomics.cas!( + x::Ptr{$typ}, + cmp::$typ, + new::$typ, + ::$(typeof(success_ordering)), + ::$(typeof(failure_ordering)), + ::$(typeof(sync)), + ) + success = Ref{Int8}() + GC.@preserve success begin + old = llvmcall( + $( + """ + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering + %rv = extractvalue { $lt, i1 } %rs, 0 + %s1 = extractvalue { $lt, i1 } %rs, 1 + %s8 = zext i1 %s1 to i8 + %sptr = inttoptr i$WORD_SIZE %3 to i8* + store i8 %s8, i8* %sptr + ret $lt %rv + """ + ), + $typ, + Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}}, + x, + cmp, + new, + Ptr{Int8}(pointer_from_objref(success)), + ) + end + return (old = old, success = !iszero(success[])) + end end end end @@ -186,60 +201,81 @@ for typ in (inttypes..., floattypes...) end end for ord in orderings - # Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged - if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE - @eval function UnsafeAtomics.modify!( + for sync in syncscopes + # Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged + if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none + @eval function UnsafeAtomics.modify!( + x::Ptr{$typ}, + op::typeof($op), + v::$typ, + ::$(typeof(ord)), + ::$(typeof(sync)), + ) + return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord)) + end + else + @eval function UnsafeAtomics.modify!( x::Ptr{$typ}, - op::typeof($op), + ::typeof($op), v::$typ, ::$(typeof(ord)), + ::$(typeof(sync)), ) - return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord)) - end - else - @eval function UnsafeAtomics.modify!( - x::Ptr{$typ}, - ::typeof($op), - v::$typ, - ::$(typeof(ord)), - ) - old = llvmcall( - $(""" - %ptr = inttoptr i$WORD_SIZE %0 to $lt* - %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord - ret $lt %rv - """), - $typ, - Tuple{Ptr{$typ},$typ}, - x, - v, - ) - return old => $op(old, v) + old = llvmcall( + $(""" + %ptr = inttoptr i$WORD_SIZE %0 to $lt* + %rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord + ret $lt %rv + """), + $typ, + Tuple{Ptr{$typ},$typ}, + x, + v, + ) + return old => $op(old, v) + end end end end end end -# Core.Intrinsics.atomic_fence was introduced in 1.10 -function UnsafeAtomics.fence(ord::Ordering) - Core.Intrinsics.atomic_fence(base_ordering(ord)) - return nothing -end -if Sys.ARCH == :x86_64 - # FIXME: Disable this once on LLVM 19 - # This is unfortunatly required for good-performance on AMD - # https://github.com/llvm/llvm-project/pull/106555 - function UnsafeAtomics.fence(::typeof(seq_cst)) - Base.llvmcall( - (raw""" - define void @fence() #0 { - entry: - tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory} - ret void - } - attributes #0 = { alwaysinline } - """, "fence"), Nothing, Tuple{}) +for sync in syncscopes + if sync == none + # Core.Intrinsics.atomic_fence was introduced in 1.10 + @eval function UnsafeAtomics.fence(ord::Ordering, ::$(typeof(sync))) + Core.Intrinsics.atomic_fence(base_ordering(ord)) + return nothing + end + if Sys.ARCH == :x86_64 + # FIXME: Disable this once on LLVM 19 + # This is unfortunatly required for good-performance on AMD + # https://github.com/llvm/llvm-project/pull/106555 + @eval function UnsafeAtomics.fence(::typeof(seq_cst), ::$(typeof(sync))) + Base.llvmcall( + (raw""" + define void @fence() #0 { + entry: + tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory} + ret void + } + attributes #0 = { alwaysinline } + """, "fence"), Nothing, Tuple{}) + end + end + else + for ord in orderings + @eval function UnsafeAtomics.fence(::$(typeof(ord)), ::$(typeof(sync))) + return llvmcall( + $(""" + fence $sync $ord + ret void + """), + Cvoid, + Tuple{}, + ) + end + end end end diff --git a/src/syncscopes.jl b/src/syncscopes.jl new file mode 100644 index 0000000..6a201c6 --- /dev/null +++ b/src/syncscopes.jl @@ -0,0 +1,16 @@ +struct LLVMSyncScope{name} <: SyncScope end + +const none = LLVMSyncScope{:none}() +const singlethread = LLVMSyncScope{:singlethread}() + +const syncscopes = (none, singlethread) +const ConcreteSyncScopes = Union{map(typeof, syncscopes)...} + +llvm_syncscope(::LLVMSyncScope{name}) where {name} = name + +Base.string(s::LLVMSyncScope) = string("syncscope(\"", llvm_syncscope(s), "\")") +Base.string(s::typeof(none)) = "" + +Base.print(io::IO, s::LLVMSyncScope) = print(io, string(s)) + +Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_ordering(o)) \ No newline at end of file