diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44a4561..1720176 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,15 +13,16 @@ jobs: strategy: matrix: julia-version: - - '1.6' - - '~1.7.0-beta4' + - 'lts' + - '1.11' + - '1.12-nightly' - 'nightly' fail-fast: false name: Test Julia ${{ matrix.julia-version }} steps: - uses: actions/checkout@v2 - name: Setup julia - uses: julia-actions/setup-julia@v1 + uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.julia-version }} - uses: julia-actions/julia-runtest@v1 @@ -31,37 +32,3 @@ jobs: file: ./lcov.info flags: unittests name: codecov-umbrella - - aqua: - runs-on: ubuntu-latest - strategy: - matrix: - julia-version: - - '1' - - '1.6' - - 'nightly' - fail-fast: false - steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 - with: - version: ${{ matrix.julia-version }} - - uses: tkf/julia-aqua@v1 - - # https://github.com/tkf/julia-code-style-suggesters - code-style: - if: always() && github.event.pull_request - runs-on: ubuntu-latest - steps: - - uses: tkf/julia-code-style-suggesters@v1 - - # A job that succeeds if and only if all jobs succeed. - all-success: - if: always() && github.event.pull_request - needs: [test, aqua, code-style] - runs-on: ubuntu-latest - steps: - # https://github.com/tkf/merge-conclusions-action - - uses: tkf/merge-conclusions-action@v1 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Project.toml b/Project.toml index 4456931..4851885 100644 --- a/Project.toml +++ b/Project.toml @@ -1,14 +1,22 @@ name = "UnsafeAtomics" uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" authors = ["Takafumi Arakaki and contributors"] -version = "0.2.2-DEV" +version = "0.3.0" + +[weakdeps] +LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" + +[extensions] +UnsafeAtomicsLLVM = ["LLVM"] [compat] -julia = "1.6" +LLVM = "8.1, 9" +julia = "1.10" [extras] +LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TestFunctionRunner = "792026f5-ac9a-4a19-adcb-47b0ce2deb5d" [targets] -test = ["Test", "TestFunctionRunner"] +test = ["Test", "TestFunctionRunner", "LLVM"] diff --git a/ext/UnsafeAtomicsLLVM/UnsafeAtomicsLLVM.jl b/ext/UnsafeAtomicsLLVM/UnsafeAtomicsLLVM.jl new file mode 100644 index 0000000..f0aca14 --- /dev/null +++ b/ext/UnsafeAtomicsLLVM/UnsafeAtomicsLLVM.jl @@ -0,0 +1,13 @@ +baremodule UnsafeAtomicsLLVM + +module Internal + +using Core: LLVMPtr +using LLVM +using UnsafeAtomics: UnsafeAtomics, Ordering + +include("internal.jl") + +end # module Internal + +end # baremodule UnsafeAtomicsLLVM diff --git a/ext/UnsafeAtomicsLLVM/atomics.jl b/ext/UnsafeAtomicsLLVM/atomics.jl new file mode 100644 index 0000000..9b1e504 --- /dev/null +++ b/ext/UnsafeAtomicsLLVM/atomics.jl @@ -0,0 +1,572 @@ +using LLVM +using LLVM.Interop + +const MEMORY_ORDERING_EXPLANATION = """ +specified as a symbol (e.g., `:sequentially_consistent`) or a `Val` of a symbol (e.g., +`Val(:sequentially_consistent)`) +""" + +""" + atomic_pointerref(pointer::LLVMPtr{T}, ordering) -> value::T +Load a `value` from `pointer` with the given memory `ordering` atomically. +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. +See also: `getproperty`, `getfield` +""" +atomic_pointerref + +""" + LLVM.Interop.atomic_pointerset(pointer::LLVMPtr{T}, x::T, ordering) -> pointer +Store a value `x` in `pointer` with the given memory `ordering` atomically. +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. +See also: `setproperty!`, `setfield!` +""" +atomic_pointerset + +""" + atomic_pointermodify( + pointer::LLVMPtr{T}, + op, + x::T, + ordering, + ) -> (old => new)::Pair{T,T} +Replace an `old` value stored at `pointer` with a `new` value comped as `new = op(old, x)` +with the given memory `ordering` atomically. Return a pair `old => new`. +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. +See also: `modifyproperty!`, `modifyfield!` +""" +atomic_pointermodify + +""" + atomic_pointerswap(pointer::LLVMPtr{T}, op, new::T, ordering) -> old::T +Replace an `old` value stored at `pointer` with a `new` value with the given memory +`ordering` atomically. Return the `old` value. +`ordering` is a Julia atomic ordering $MEMORY_ORDERING_EXPLANATION. +See also: `modifyproperty!`, `modifyfield!` +""" +atomic_pointerswap + +""" + atomic_pointerreplace( + pointer::LLVMPtr{T}, + expected::T, + desired::T, + success_ordering, + fail_ordering, + ) -> (; old::T, success::Bool) +Try to replace the value of `pointer` from an `expected` value to a `desired` value +atomically with the ordering `success_ordering`. The property `old` of the returned value +is the value stored in the `pointer`. The property `success` of the returned value +indicates if the replacement was successful. The ordering `fail_ordering` specifies the +ordering used for loading the `old` value. +`success_ordering` and `fail_ordering` are Julia atomic orderings +$MEMORY_ORDERING_EXPLANATION. +See also: `replaceproperty!`, `replacefield!` +""" +atomic_pointerreplace + +const _llvm_from_julia_ordering = ( + not_atomic = LLVM.API.LLVMAtomicOrderingNotAtomic, + unordered = LLVM.API.LLVMAtomicOrderingUnordered, + monotonic = LLVM.API.LLVMAtomicOrderingMonotonic, + acquire = LLVM.API.LLVMAtomicOrderingAcquire, + release = LLVM.API.LLVMAtomicOrderingRelease, + acquire_release = LLVM.API.LLVMAtomicOrderingAcquireRelease, + sequentially_consistent = LLVM.API.LLVMAtomicOrderingSequentiallyConsistent, +) + +_julia_ordering(p) = + Union{map(x -> p(x) ? Val{x} : Union{}, keys(_llvm_from_julia_ordering))...} + +const AllOrdering = _julia_ordering(_ -> true) +const AtomicOrdering = _julia_ordering(!=(:not_atomic)) + +const LLVMOrderingVal = Union{map(x -> Val{x}, values(_llvm_from_julia_ordering))...} + +is_stronger_than_monotonic(order::Symbol) = + !(order === :monotonic || order === :unordered || order === :not_atomic) + +for (julia, llvm) in pairs(_llvm_from_julia_ordering) + @eval llvm_from_julia_ordering(::Val{$(QuoteNode(julia))}) = Val{$llvm}() +end + +""" + @dynamic_order(order) do order + ... use order ... + end +It is expanded to an expression similar to: + if order === :not_atomic + let order = Val(:not_atomic) + ... use order ... + end + elseif order === :unordered + let order = Val(:unordered) + ... use order ... + end + elseif ... + ... + else + throw(ConcurrencyViolationError(...)) + end +This is used for helping the compiler to optimize expressions such as +`atomic_pointerref(ptr, :monotonic)` and also to avoid abstract run-time dispatch. +""" +macro dynamic_order(thunk, order) + @assert Meta.isexpr(thunk, :->, 2) && Meta.isexpr(thunk.args[1], :tuple, 1) + ordervar = esc(thunk.args[1].args[1]) + body = esc(thunk.args[2]) + expr = foldr( + keys(_llvm_from_julia_ordering), + init = :(throw(ConcurrencyViolationError("invalid atomic ordering: ", order))), + ) do key, r + quote + if order === $(QuoteNode(key)) + let $ordervar = Val{$(QuoteNode(key))}() + $body + end + else + $r + end + end + end + quote + order = $(esc(order)) + $expr + end +end + +_valueof(::Val{x}) where {x} = x + +@inline function atomic_pointerref(pointer, order::Symbol) + @dynamic_order(order) do order + atomic_pointerref(pointer, order) + end +end + +@inline function atomic_pointerset(pointer, x, order::Symbol) + @dynamic_order(order) do order + atomic_pointerset(pointer, x, order) + end +end + +@generated function atomic_pointerref(ptr::LLVMPtr{T,A}, order::AllOrdering) where {T,A} + sizeof(T) == 0 && return T.instance + llvm_order = _valueof(llvm_from_julia_ordering(order())) + @dispose ctx = Context() begin + eltyp = convert(LLVMType, T) + + T_ptr = convert(LLVMType, ptr) + + T_typed_ptr = LLVM.PointerType(eltyp, A) + + # create a function + param_types = [T_ptr] + llvm_f, _ = create_function(eltyp, param_types) + + # generate IR + @dispose builder = IRBuilder() begin + entry = BasicBlock(llvm_f, "entry") + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + ld = load!(builder, eltyp, typed_ptr) + ordering!(ld, llvm_order) + + if A != 0 + metadata(ld)[LLVM.MD_tbaa] = tbaa_addrspace(A) + end + alignment!(ld, sizeof(T)) + + ret!(builder, ld) + end + + call_function(llvm_f, T, Tuple{LLVMPtr{T,A}}, :ptr) + end +end + +@generated function atomic_pointerset( + ptr::LLVMPtr{T,A}, + x::T, + order::AllOrdering, +) where {T,A} + if sizeof(T) == 0 + # Mimicking what `Core.Intrinsics.atomic_pointerset` generates. + # See: https://github.com/JuliaLang/julia/blob/v1.7.2/src/cgutils.cpp#L1570-L1572 + is_stronger_than_monotonic(order) || return :ptr + return quote + Core.Intrinsics.fence($(QuoteNode(order))) + ptr + end + end + llvm_order = _valueof(llvm_from_julia_ordering(order())) + @dispose ctx = Context() begin + eltyp = convert(LLVMType, T) + T_ptr = convert(LLVMType, ptr) + T_typed_ptr = LLVM.PointerType(eltyp, A) + + # create a function + param_types = [T_ptr, eltyp] + llvm_f, _ = create_function(LLVM.VoidType(), param_types) + + # generate IR + @dispose builder = IRBuilder() begin + entry = BasicBlock(llvm_f, "entry") + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + val = parameters(llvm_f)[2] + st = store!(builder, val, typed_ptr) + ordering!(st, llvm_order) + + if A != 0 + metadata(st)[LLVM.MD_tbaa] = tbaa_addrspace(A) + end + alignment!(st, sizeof(T)) + + ret!(builder) + end + + call = call_function(llvm_f, Cvoid, Tuple{LLVMPtr{T,A},T}, :ptr, :x) + quote + $call + ptr + end + end +end + +right(_, r) = r + +const binoptable = [ + (:xchg, right, LLVM.API.LLVMAtomicRMWBinOpXchg), + (:add, +, LLVM.API.LLVMAtomicRMWBinOpAdd), + (:sub, -, LLVM.API.LLVMAtomicRMWBinOpSub), + (:and, &, LLVM.API.LLVMAtomicRMWBinOpAnd), + (:or, |, LLVM.API.LLVMAtomicRMWBinOpOr), + (:xor, xor, LLVM.API.LLVMAtomicRMWBinOpXor), + (:max, max, LLVM.API.LLVMAtomicRMWBinOpMax), + (:min, min, LLVM.API.LLVMAtomicRMWBinOpMin), + (:umax, max, LLVM.API.LLVMAtomicRMWBinOpUMax), + (:umin, min, LLVM.API.LLVMAtomicRMWBinOpUMin), + (:fadd, +, LLVM.API.LLVMAtomicRMWBinOpFAdd), + (:fsub, -, LLVM.API.LLVMAtomicRMWBinOpFSub), + (:fmax, max, LLVM.API.LLVMAtomicRMWBinOpFMax), + (:fmin, min, LLVM.API.LLVMAtomicRMWBinOpFMin), +] + +const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} + +# LLVM API accepts string literal as a syncscope argument. +@inline syncscope_to_string(::Type{Val{S}}) where {S} = string(S) + +@generated function llvm_atomic_op( + binop::AtomicRMWBinOpVal, + ptr::LLVMPtr{T,A}, + val::T, + order::LLVMOrderingVal, +) where {T,A} + @dispose ctx = Context() begin + T_val = convert(LLVMType, T) + T_ptr = convert(LLVMType, ptr) + + T_typed_ptr = LLVM.PointerType(T_val, A) + + llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + + @dispose builder = IRBuilder() begin + entry = BasicBlock(llvm_f, "entry") + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + + single_threaded = false + rv = atomic_rmw!( + builder, + _valueof(binop()), + typed_ptr, + parameters(llvm_f)[2], + _valueof(order()), + single_threaded, + ) + + ret!(builder, rv) + end + + call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) + end +end + +@generated function llvm_atomic_op( + binop::AtomicRMWBinOpVal, + ptr::LLVMPtr{T,A}, + val::T, + order::LLVMOrderingVal, + syncscope::Val{S}, +) where {T,A,S} + @dispose ctx = Context() begin + T_val = convert(LLVMType, T) + T_ptr = convert(LLVMType, ptr) + + T_typed_ptr = LLVM.PointerType(T_val, A) + llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + + @dispose builder = IRBuilder() begin + entry = BasicBlock(llvm_f, "entry") + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + rv = atomic_rmw!( + builder, + _valueof(binop()), + typed_ptr, + parameters(llvm_f)[2], + _valueof(order()), + syncscope_to_string(syncscope), + ) + + ret!(builder, rv) + end + call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) + end +end + +@inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP} + @dynamic_order(order) do order + atomic_pointermodify(pointer, op, x, order) + end +end + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + op, + x::T, + ::Val{:not_atomic}, +) where {T} + old = atomic_pointerref(ptr, Val(:not_atomic)) + new = op(old, x) + atomic_pointerset(ptr, new, Val(:not_atomic)) + return old => new +end + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + ::typeof(right), + x::T, + order::AtomicOrdering, +) where {T} + old = llvm_atomic_op( + Val(LLVM.API.LLVMAtomicRMWBinOpXchg), + ptr, + x, + llvm_from_julia_ordering(order), + ) + return old => x +end + +const atomictypes = Any[ + Int8, + Int16, + Int32, + Int64, + Int128, + UInt8, + UInt16, + UInt32, + UInt64, + UInt128, + Float16, + Float32, + Float64, +] + +for (opname, op, llvmop) in binoptable + opname === :xchg && continue + types = if opname in (:min, :max) + filter(t -> t <: Signed, atomictypes) + elseif opname in (:umin, :umax) + filter(t -> t <: Unsigned, atomictypes) + elseif opname in (:fadd, :fsub, :fmin, :fmax) + filter(t -> t <: AbstractFloat, atomictypes) + else + filter(t -> t <: Integer, atomictypes) + end + for T in types + @eval @inline function atomic_pointermodify( + ptr::LLVMPtr{$T}, + ::$(typeof(op)), + x::$T, + order::AtomicOrdering, + syncscope::Val{S} = Val{:system}(), + ) where {S} + old = + syncscope isa Val{:system} ? + llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : + llvm_atomic_op( + $(Val(llvmop)), + ptr, + x, + llvm_from_julia_ordering(order), + syncscope, + ) + return old => $op(old, x) + end + end +end + +@inline atomic_pointerswap(pointer, new) = first(atomic_pointermodify(pointer, right, new)) +@inline atomic_pointerswap(pointer, new, order) = + first(atomic_pointermodify(pointer, right, new, order)) + +@inline function atomic_pointermodify( + ptr::LLVMPtr{T}, + op, + x::T, + order::AllOrdering, +) where {T} + # Should `fail_order` be stronger? Ref: https://github.com/JuliaLang/julia/issues/45256 + fail_order = Val(:monotonic) + old = atomic_pointerref(ptr, fail_order) + while true + new = op(old, x) + (old, success) = atomic_pointerreplace(ptr, old, new, order, fail_order) + success && return old => new + end +end + +@generated function llvm_atomic_cas( + ptr::LLVMPtr{T,A}, + cmp::T, + val::T, + success_order::LLVMOrderingVal, + fail_order::LLVMOrderingVal, +) where {T,A} + llvm_success = _valueof(success_order()) + llvm_fail = _valueof(fail_order()) + @dispose ctx = Context() begin + T_val = convert(LLVMType, T) + T_pointee = T_val + if T_val isa LLVM.FloatingPointType + T_pointee = LLVM.IntType(sizeof(T) * 8) + end + T_ptr = convert(LLVMType, ptr) + T_success = convert(LLVMType, Ptr{Int8}) + + T_typed_ptr = LLVM.PointerType(T_pointee, A) + T_ok_ptr = LLVM.PointerType(convert(LLVMType, Int8)) + + llvm_f, _ = create_function(T_val, [T_ptr, T_val, T_val, T_success]) + + @dispose builder = IRBuilder() begin + entry = BasicBlock(llvm_f, "entry") + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + ok_ptr = inttoptr!(builder, parameters(llvm_f)[4], T_ok_ptr) + + cmp_int = parameters(llvm_f)[2] + if T_val isa LLVM.FloatingPointType + cmp_int = bitcast!(builder, cmp_int, T_pointee) + end + + val_int = parameters(llvm_f)[3] + if T_val isa LLVM.FloatingPointType + val_int = bitcast!(builder, val_int, T_pointee) + end + + single_threaded = false + res = atomic_cmpxchg!( + builder, + typed_ptr, + cmp_int, + val_int, + llvm_success, + llvm_fail, + single_threaded, + ) + + rv = extract_value!(builder, res, 0) + ok = extract_value!(builder, res, 1) + ok = zext!(builder, ok, LLVM.Int8Type()) + store!(builder, ok, ok_ptr) + + if T_val isa LLVM.FloatingPointType + rv = bitcast!(builder, rv, T_val) + end + + ret!(builder, rv) + end + + expr = call_function( + llvm_f, + T, + Tuple{LLVMPtr{T,A},T,T,Ptr{Int8}}, + :ptr, + :cmp, + :val, + :success_ptr, + ) + quote + success = Ref{Int8}() + old = GC.@preserve success begin + success_ptr = Ptr{Int8}(pointer_from_objref(success)) + $expr + end + (; old, success = success[] != zero(Int8)) + end + end +end + +@inline function atomic_pointerreplace( + pointer, + expected, + desired, + success_order::Symbol, + fail_order::Symbol, +) + # This avoids abstract dispatch at run-time but probably too much codegen? + #= + @dynamic_order(success_order) do success_order + @dynamic_order(fail_order) do fail_order + atomic_pointerreplace(pointer, expected, desired, success_order, fail_order) + end + end + =# + + # This avoids excessive codegen while hopefully imposes no cost when const-prop works: + so = @dynamic_order(success_order) do success_order + success_order + end + fo = @dynamic_order(fail_order) do fail_order + fail_order + end + return atomic_pointerreplace(pointer, expected, desired, so, fo) +end + +@inline function atomic_pointerreplace( + ptr::LLVMPtr{T}, + expected::T, + desired::T, + ::Val{:not_atomic}, + ::Val{:not_atomic}, +) where {T} + old = atomic_pointerref(ptr, Val(:not_atomic)) + if old === expected + atomic_pointerset(ptr, desired, Val(:not_atomic)) + success = true + else + success = false + end + return (; old, success) +end + +@inline atomic_pointerreplace( + ptr::LLVMPtr{T}, + expected::T, + desired::T, + success_order::_julia_ordering(∉((:not_atomic, :unordered))), + fail_order::_julia_ordering(∉((:not_atomic, :unordered, :release, :acquire_release))), +) where {T} = llvm_atomic_cas( + ptr, + expected, + desired, + llvm_from_julia_ordering(success_order), + llvm_from_julia_ordering(fail_order), +) diff --git a/ext/UnsafeAtomicsLLVM/internal.jl b/ext/UnsafeAtomicsLLVM/internal.jl new file mode 100644 index 0000000..e4ed556 --- /dev/null +++ b/ext/UnsafeAtomicsLLVM/internal.jl @@ -0,0 +1,44 @@ +# TODO: move this to UnsafeAtomics +julia_ordering_name(::UnsafeAtomics.Internal.LLVMOrdering{name}) where {name} = name +julia_ordering_name(::typeof(UnsafeAtomics.acquire_release)) = :acquire_release +julia_ordering_name(::typeof(UnsafeAtomics.sequentially_consistent)) = + :sequentially_consistent + +include("atomics.jl") + +@inline UnsafeAtomics.load(ptr::LLVMPtr, order::Ordering) = + atomic_pointerref(ptr, Val{julia_ordering_name(order)}()) + +@inline function UnsafeAtomics.store!(ptr::LLVMPtr, x, order::Ordering) + atomic_pointerset(ptr, x, Val{julia_ordering_name(order)}()) + return +end + +mapop(op::OP) where {OP} = op +mapop(::typeof(UnsafeAtomics.right)) = right + +@inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering) where {OP} = + atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}()) + +@inline UnsafeAtomics.modify!( + ptr::LLVMPtr, + op::OP, + x, + order::Ordering, + syncscope::Val{S} = Val(:system), +) where {OP<:Union{typeof(+),typeof(-)},S} = + atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope) + +@inline UnsafeAtomics.cas!( + ptr::LLVMPtr, + expected, + desired, + success_order::Ordering, + failure_order::Ordering, +) = atomic_pointerreplace( + ptr, + expected, + desired, + Val{julia_ordering_name(success_order)}(), + Val{julia_ordering_name(failure_order)}(), +) diff --git a/test/UnsafeAtomicsLLVM.jl b/test/UnsafeAtomicsLLVM.jl new file mode 100644 index 0000000..5eec490 --- /dev/null +++ b/test/UnsafeAtomicsLLVM.jl @@ -0,0 +1,82 @@ +import LLVM + +using UnsafeAtomics: UnsafeAtomics, acquire, release, acq_rel, seq_cst +using UnsafeAtomics.Internal: OP_RMW_TABLE, inttypes +using Test + +llvmptr(xs::Array, i) = reinterpret(Core.LLVMPtr{eltype(xs),0}, pointer(xs, i)) + +function check_default_ordering(T::Type) + xs = T[rand(T), rand(T)] + x1 = rand(T) + x2 = rand(T) + @debug "xs=$(repr(xs)) x1=$(repr(x1)) x2=$(repr(x2))" + + ptr = llvmptr(xs, 1) + GC.@preserve xs begin + @test UnsafeAtomics.load(ptr) === xs[1] + UnsafeAtomics.store!(ptr, x1) + @test xs[1] === x1 + desired = (old = x1, success = true) + @test UnsafeAtomics.cas!(ptr, x1, x2) === (old = x1, success = true) + @test xs[1] === x2 + @testset for (op, name) in OP_RMW_TABLE + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + + rmw = getfield(UnsafeAtomics, Symbol(name, :!)) + xs[1] = x1 + @test rmw(ptr, x2) === x1 + @test xs[1] === op(x1, x2) + end + end +end + +function test_explicit_ordering(T::Type = UInt) + xs = T[rand(T), rand(T)] + x1 = rand(T) + x2 = rand(T) + @debug "xs=$(repr(xs)) x1=$(repr(x1)) x2=$(repr(x2))" + + ptr = llvmptr(xs, 1) + GC.@preserve xs begin + + @test UnsafeAtomics.load(ptr, acquire) === xs[1] + UnsafeAtomics.store!(ptr, x1, release) + @test xs[1] === x1 + desired = (old = x1, success = true) + @test UnsafeAtomics.cas!(ptr, x1, x2, acq_rel, acquire) === desired + @test xs[1] === x2 + @testset for (op, name) in OP_RMW_TABLE + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2, acq_rel) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + + rmw = getfield(UnsafeAtomics, Symbol(name, :!)) + xs[1] = x1 + @test rmw(ptr, x2, acquire) === x1 + @test xs[1] === op(x1, x2) + + # Test syncscopes. + if (op == +) || (op == -) + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:system)) === + (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:singlethread)) === + (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + end + end + end +end + +@testset "UnsafeAtomicsLLVM" begin + @testset for T in inttypes + check_default_ordering(T) + test_explicit_ordering(T) + end +end diff --git a/test/runtests.jl b/test/runtests.jl index d358b79..2b9f003 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,2 +1,4 @@ using TestFunctionRunner TestFunctionRunner.@run + +include("UnsafeAtomicsLLVM.jl") \ No newline at end of file