Skip to content

Commit

Permalink
implement syncscopes everywhere
Browse files Browse the repository at this point in the history
  • Loading branch information
@vchuravy
vchuravy committed Dec 4, 2024
1 parent b099010 commit 828833a
Show file tree
Hide file tree
Showing 3 changed files with 182 additions and 124 deletions.
8 changes: 7 additions & 1 deletion src/UnsafeAtomics.jl
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
baremodule UnsafeAtomics

abstract type Ordering end
abstract type SyncScope end

function load end
function store! end
Expand All @@ -26,10 +27,11 @@ module Internal
using Base.Sys: WORD_SIZE
using Base: bitcast, llvmcall

using ..UnsafeAtomics: UnsafeAtomics, Ordering, right
using ..UnsafeAtomics: UnsafeAtomics, Ordering, SyncScope, right

include("utils.jl")
include("orderings.jl")
include("syncscopes.jl")
include("core.jl")

end # module Internal
Expand All @@ -45,4 +47,8 @@ const seq_cst = Internal.seq_cst
const acquire_release = acq_rel
const sequentially_consistent = seq_cst

# SyncScope
const none = Internal.none
const singlethread = Internal.singlethread

end # baremodule UnsafeAtomics
282 changes: 159 additions & 123 deletions src/core.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@
@inline UnsafeAtomics.modify!(ptr, op, x) = UnsafeAtomics.modify!(ptr, op, x, seq_cst)
@inline UnsafeAtomics.fence() = UnsafeAtomics.fence(seq_cst)

@inline UnsafeAtomics.load(x, ord) = UnsafeAtomics.load(x, ord, none)
@inline UnsafeAtomics.store!(x, v, ord) = UnsafeAtomics.store!(x, v, ord, none)
@inline UnsafeAtomics.cas!(x, cmp, new, ord) = UnsafeAtomics.cas!(x, cmp, new, ord, ord, none)
@inline UnsafeAtomics.modify!(ptr, op, x, ord) = UnsafeAtomics.modify!(ptr, op, x, ord, none)
@inline UnsafeAtomics.fence(ord) = UnsafeAtomics.fence(ord, none)

#! format: off
# https://github.com/JuliaLang/julia/blob/v1.6.3/base/atomics.jl#L23-L30
if Sys.ARCH == :i686 || startswith(string(Sys.ARCH), "arm") ||
Expand Down Expand Up @@ -45,8 +51,9 @@ const OP_RMW_TABLE = [
for (op, rmwop) in OP_RMW_TABLE
fn = Symbol(rmwop, "!")
@eval @inline UnsafeAtomics.$fn(x, v) = UnsafeAtomics.$fn(x, v, seq_cst)
@eval @inline UnsafeAtomics.$fn(ptr, x, ord) =
first(UnsafeAtomics.modify!(ptr, $op, x, ord))
@eval @inline UnsafeAtomics.$fn(x, v, ord) = UnsafeAtomics.$fn(x, v, ord, none)
@eval @inline UnsafeAtomics.$fn(ptr, x, ord, scope) =
first(UnsafeAtomics.modify!(ptr, $op, x, ord, scope))
end

const ATOMIC_INTRINSICS = isdefined(Core.Intrinsics, :atomic_pointerref)
Expand All @@ -67,47 +74,51 @@ for typ in (inttypes..., floattypes...)
for ord in orderings
ord in (release, acq_rel) && continue

if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
end
else
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ}},
x,
)
for sync in syncscopes
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync)))
return Core.Intrinsics.atomic_pointerref(x, base_ordering($ord))
end
else
@eval function UnsafeAtomics.load(x::Ptr{$typ}, ::$(typeof(ord)), ::$(typeof(sync)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = load atomic $rt %ptr $ord, align $(sizeof(typ))
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ}},
x,
)
end
end
end
end

for ord in orderings
ord in (acquire, acq_rel) && continue

if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
return nothing
end
else
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
ret void
"""),
Cvoid,
Tuple{Ptr{$typ},$typ},
x,
v,
)

for sync in syncscopes
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync)))
Core.Intrinsics.atomic_pointerset(x, v, base_ordering($ord))
return nothing
end
else
@eval function UnsafeAtomics.store!(x::Ptr{$typ}, v::$typ, ::$(typeof(ord)), ::$(typeof(sync)))
return llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
store atomic $lt %1, $lt* %ptr $ord, align $(sizeof(typ))
ret void
"""),
Cvoid,
Tuple{Ptr{$typ},$typ},
x,
v,
)
end
end
end
end
Expand All @@ -117,54 +128,58 @@ for typ in (inttypes..., floattypes...)

typ <: AbstractFloat && break

if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
)
return Core.Intrinsics.atomic_pointerreplace(
x,
cmp,
new,
base_ordering($success_ordering),
base_ordering($failure_ordering)
for sync in syncscopes
if ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
::$(typeof(sync)),
)
end
else
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
)
success = Ref{Int8}()
GC.@preserve success begin
old = llvmcall(
$(
"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
%rv = extractvalue { $lt, i1 } %rs, 0
%s1 = extractvalue { $lt, i1 } %rs, 1
%s8 = zext i1 %s1 to i8
%sptr = inttoptr i$WORD_SIZE %3 to i8*
store i8 %s8, i8* %sptr
ret $lt %rv
"""
),
$typ,
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
return Core.Intrinsics.atomic_pointerreplace(
x,
cmp,
new,
Ptr{Int8}(pointer_from_objref(success)),
base_ordering($success_ordering),
base_ordering($failure_ordering)
)
end
return (old = old, success = !iszero(success[]))
else
@eval function UnsafeAtomics.cas!(
x::Ptr{$typ},
cmp::$typ,
new::$typ,
::$(typeof(success_ordering)),
::$(typeof(failure_ordering)),
::$(typeof(sync)),
)
success = Ref{Int8}()
GC.@preserve success begin
old = llvmcall(
$(
"""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rs = cmpxchg $lt* %ptr, $lt %1, $lt %2 $success_ordering $failure_ordering
%rv = extractvalue { $lt, i1 } %rs, 0
%s1 = extractvalue { $lt, i1 } %rs, 1
%s8 = zext i1 %s1 to i8
%sptr = inttoptr i$WORD_SIZE %3 to i8*
store i8 %s8, i8* %sptr
ret $lt %rv
"""
),
$typ,
Tuple{Ptr{$typ},$typ,$typ,Ptr{Int8}},
x,
cmp,
new,
Ptr{Int8}(pointer_from_objref(success)),
)
end
return (old = old, success = !iszero(success[]))
end
end
end
end
Expand All @@ -186,60 +201,81 @@ for typ in (inttypes..., floattypes...)
end
end
for ord in orderings
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE
@eval function UnsafeAtomics.modify!(
for sync in syncscopes
# Enable this code iff https://github.com/JuliaLang/julia/pull/45122 get's merged
if false && ATOMIC_INTRINSICS && sizeof(typ) <= MAX_POINTERATOMIC_SIZE && sync == none
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
op::typeof($op),
v::$typ,
::$(typeof(ord)),
::$(typeof(sync)),
)
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
end
else
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
op::typeof($op),
::typeof($op),
v::$typ,
::$(typeof(ord)),
::$(typeof(sync)),
)
return Core.Intrinsics.atomic_pointermodify(x, op, v, base_ordering($ord))
end
else
@eval function UnsafeAtomics.modify!(
x::Ptr{$typ},
::typeof($op),
v::$typ,
::$(typeof(ord)),
)
old = llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ},$typ},
x,
v,
)
return old => $op(old, v)
old = llvmcall(
$("""
%ptr = inttoptr i$WORD_SIZE %0 to $lt*
%rv = atomicrmw $rmw $lt* %ptr, $lt %1 $ord
ret $lt %rv
"""),
$typ,
Tuple{Ptr{$typ},$typ},
x,
v,
)
return old => $op(old, v)
end
end
end
end
end
end

# Core.Intrinsics.atomic_fence was introduced in 1.10
function UnsafeAtomics.fence(ord::Ordering)
Core.Intrinsics.atomic_fence(base_ordering(ord))
return nothing
end
if Sys.ARCH == :x86_64
# FIXME: Disable this once on LLVM 19
# This is unfortunatly required for good-performance on AMD
# https://github.com/llvm/llvm-project/pull/106555
function UnsafeAtomics.fence(::typeof(seq_cst))
Base.llvmcall(
(raw"""
define void @fence() #0 {
entry:
tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory}
ret void
}
attributes #0 = { alwaysinline }
""", "fence"), Nothing, Tuple{})
for sync in syncscopes
if sync == none
# Core.Intrinsics.atomic_fence was introduced in 1.10
@eval function UnsafeAtomics.fence(ord::Ordering, ::$(typeof(sync)))
Core.Intrinsics.atomic_fence(base_ordering(ord))
return nothing
end
if Sys.ARCH == :x86_64
# FIXME: Disable this once on LLVM 19
# This is unfortunatly required for good-performance on AMD
# https://github.com/llvm/llvm-project/pull/106555
@eval function UnsafeAtomics.fence(::typeof(seq_cst), ::$(typeof(sync)))
Base.llvmcall(
(raw"""
define void @fence() #0 {
entry:
tail call void asm sideeffect "lock orq $$0 , (%rsp)", ""(); should this have ~{memory}
ret void
}
attributes #0 = { alwaysinline }
""", "fence"), Nothing, Tuple{})
end
end
else
for ord in orderings
@eval function UnsafeAtomics.fence(::$(typeof(ord)), ::$(typeof(sync)))
return llvmcall(
$("""
fence $sync $ord
ret void
"""),
Cvoid,
Tuple{},
)
end
end
end
end

Expand Down
16 changes: 16 additions & 0 deletions src/syncscopes.jl
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
struct LLVMSyncScope{name} <: SyncScope end

const none = LLVMSyncScope{:none}()
const singlethread = LLVMSyncScope{:singlethread}()

const syncscopes = (none, singlethread)
const ConcreteSyncScopes = Union{map(typeof, syncscopes)...}

llvm_syncscope(::LLVMSyncScope{name}) where {name} = name

Base.string(s::LLVMSyncScope) = string("syncscope(\"", llvm_syncscope(s), "\")")
Base.string(s::typeof(none)) = ""

Base.print(io::IO, s::LLVMSyncScope) = print(io, string(s))

Base.show(io::IO, o::ConcreteSyncScopes) = print(io, UnsafeAtomics, '.', llvm_ordering(o))

0 comments on commit 828833a

Please sign in to comment.