From fe16aa4b1f587fb98b6da80079444a2a49280e1a Mon Sep 17 00:00:00 2001 From: Anton Smirnov Date: Thu, 25 Jul 2024 14:58:42 +0300 Subject: [PATCH 1/6] Add syncscope --- LocalPreferences.toml | 2 ++ src/atomics.jl | 53 ++++++++++++++++++++++++++++++++++++++++++- src/internal.jl | 6 +++++ test/runtests.jl | 17 ++++++++++++-- 4 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 LocalPreferences.toml diff --git a/LocalPreferences.toml b/LocalPreferences.toml new file mode 100644 index 0000000..2e003b0 --- /dev/null +++ b/LocalPreferences.toml @@ -0,0 +1,2 @@ +[LLVM] +libLLVMExtra = "/home/pxlth/.julia/scratchspaces/929cbde3-209d-540e-8aea-75f648917ca0/build/lib/libLLVMExtra-16.so" diff --git a/src/atomics.jl b/src/atomics.jl index 823e60f..4797fcf 100644 --- a/src/atomics.jl +++ b/src/atomics.jl @@ -256,6 +256,21 @@ end const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} +# Syncscopes. +const SyncscopeSystem = Val{:system}() +const SyncscopeSingleThread = Val{:singlethread}() +# AMDGPU-specific to enable fast hardware floating-point atomics: +# https://llvm.org/docs/AMDGPUUsage.html#memory-scopes +const SyncscopeAgent = Val{:agent}() + +const Syncscopes = Union{ + typeof(SyncscopeSystem), + typeof(SyncscopeSingleThread), + typeof(SyncscopeAgent)} + +# LLVM API accepts string literal as a syncscope argument. +@inline syncscope_to_string(::Type{Val{S}}) where S = string(S) + @generated function llvm_atomic_op( binop::AtomicRMWBinOpVal, ptr::LLVMPtr{T,A}, @@ -293,6 +308,39 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} end end +@generated function llvm_atomic_op( + binop::AtomicRMWBinOpVal, + ptr::LLVMPtr{T,A}, + val::T, + order::LLVMOrderingVal, + syncscope::Syncscopes, +) where {T,A} + @dispose ctx = Context() begin + T_val = convert(LLVMType, T) + T_ptr = convert(LLVMType, ptr) + + T_typed_ptr = LLVM.PointerType(T_val, A) + llvm_f, _ = create_function(T_val, [T_ptr, T_val]) + + @dispose builder = IRBuilder() begin + entry = BasicBlock(llvm_f, "entry") + position!(builder, entry) + + typed_ptr = bitcast!(builder, parameters(llvm_f)[1], T_typed_ptr) + rv = atomic_rmw!( + builder, + _valueof(binop()), + typed_ptr, + parameters(llvm_f)[2], + _valueof(order()), + syncscope_to_string(syncscope)) + + ret!(builder, rv) + end + call_function(llvm_f, T, Tuple{LLVMPtr{T,A},T}, :ptr, :val) + end +end + @inline function atomic_pointermodify(pointer, op::OP, x, order::Symbol) where {OP} @dynamic_order(order) do order atomic_pointermodify(pointer, op, x, order) @@ -359,8 +407,11 @@ for (opname, op, llvmop) in binoptable ::$(typeof(op)), x::$T, order::AtomicOrdering, + syncscope::Syncscopes = SyncscopeSystem, ) - old = llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) + old = syncscope isa typeof(SyncscopeSystem) ? + llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : + llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order), syncscope) return old => $op(old, x) end end diff --git a/src/internal.jl b/src/internal.jl index caaa84e..3f18bc4 100644 --- a/src/internal.jl +++ b/src/internal.jl @@ -20,6 +20,12 @@ mapop(::typeof(UnsafeAtomics.right)) = right @inline UnsafeAtomics.modify!(ptr::LLVMPtr, op::OP, x, order::Ordering) where {OP} = atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}()) +@inline UnsafeAtomics.modify!( + ptr::LLVMPtr, op::OP, x, order::Ordering, syncscope::Syncscopes = SyncscopeSystem, +) where {OP <: Union{typeof(+), typeof(-)}} = atomic_pointermodify( + ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope, +) + @inline UnsafeAtomics.cas!( ptr::LLVMPtr, expected, diff --git a/test/runtests.jl b/test/runtests.jl index b27b351..8df89e8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,6 @@ import UnsafeAtomicsLLVM -using UnsafeAtomics: UnsafeAtomics, acquire, release, acq_rel +using UnsafeAtomics: UnsafeAtomics, acquire, release, acq_rel, seq_cst using UnsafeAtomics.Internal: OP_RMW_TABLE, inttypes using Test @@ -29,6 +29,18 @@ function check_default_ordering(T::Type) xs[1] = x1 @test rmw(ptr, x2) === x1 @test xs[1] === op(x1, x2) + + # Test syncscopes. + if (op == +) || (op == -) + @info "!!!!!!!!!!!!!!!!!!!" + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomicsLLVM.Internal.SyncscopeSystem) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + + xs[1] = x1 + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomicsLLVM.Internal.SyncscopeSingleThread) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + end end end end @@ -64,6 +76,7 @@ end @testset "UnsafeAtomicsLLVM" begin @testset for T in inttypes check_default_ordering(T) - test_explicit_ordering(T) + # test_explicit_ordering(T) + break end end From e5aa841628ebaaf491695cff857dcc5a6595cdc6 Mon Sep 17 00:00:00 2001 From: Anton Smirnov Date: Thu, 1 Aug 2024 11:49:25 +0300 Subject: [PATCH 2/6] Refactor --- .github/workflows/ci.yml | 6 ++---- LocalPreferences.toml | 2 -- Project.toml | 6 +++--- src/atomics.jl | 24 ++++++++++++++---------- src/internal.jl | 11 +++++++---- test/runtests.jl | 38 ++++++++++++++++++++++++-------------- 6 files changed, 50 insertions(+), 37 deletions(-) delete mode 100644 LocalPreferences.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5207671..4df1690 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,7 @@ jobs: strategy: matrix: julia-version: - - '1' - - '1.6' + - '1.10' - 'nightly' fail-fast: false name: Test Julia ${{ matrix.julia-version }} @@ -36,8 +35,7 @@ jobs: strategy: matrix: julia-version: - - '1' - - '1.6' + - '1.10' - 'nightly' fail-fast: false steps: diff --git a/LocalPreferences.toml b/LocalPreferences.toml deleted file mode 100644 index 2e003b0..0000000 --- a/LocalPreferences.toml +++ /dev/null @@ -1,2 +0,0 @@ -[LLVM] -libLLVMExtra = "/home/pxlth/.julia/scratchspaces/929cbde3-209d-540e-8aea-75f648917ca0/build/lib/libLLVMExtra-16.so" diff --git a/Project.toml b/Project.toml index 48d4011..f0d04dc 100644 --- a/Project.toml +++ b/Project.toml @@ -1,13 +1,13 @@ name = "UnsafeAtomicsLLVM" uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" authors = ["Takafumi Arakaki and contributors"] -version = "0.1.5" +version = "0.2.0" [deps] LLVM = "929cbde3-209d-540e-8aea-75f648917ca0" UnsafeAtomics = "013be700-e6cd-48c3-b4a1-df204f14c38f" [compat] -LLVM = "6, 7, 8" +LLVM = "8.1" UnsafeAtomics = "0.2" -julia = "1.6" +julia = "1.10" diff --git a/src/atomics.jl b/src/atomics.jl index 4797fcf..fd48f71 100644 --- a/src/atomics.jl +++ b/src/atomics.jl @@ -259,17 +259,13 @@ const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} # Syncscopes. const SyncscopeSystem = Val{:system}() const SyncscopeSingleThread = Val{:singlethread}() -# AMDGPU-specific to enable fast hardware floating-point atomics: -# https://llvm.org/docs/AMDGPUUsage.html#memory-scopes const SyncscopeAgent = Val{:agent}() -const Syncscopes = Union{ - typeof(SyncscopeSystem), - typeof(SyncscopeSingleThread), - typeof(SyncscopeAgent)} +const Syncscopes = + Union{typeof(SyncscopeSystem),typeof(SyncscopeSingleThread),typeof(SyncscopeAgent)} # LLVM API accepts string literal as a syncscope argument. -@inline syncscope_to_string(::Type{Val{S}}) where S = string(S) +@inline syncscope_to_string(::Type{Val{S}}) where {S} = string(S) @generated function llvm_atomic_op( binop::AtomicRMWBinOpVal, @@ -333,7 +329,8 @@ end typed_ptr, parameters(llvm_f)[2], _valueof(order()), - syncscope_to_string(syncscope)) + syncscope_to_string(syncscope), + ) ret!(builder, rv) end @@ -409,9 +406,16 @@ for (opname, op, llvmop) in binoptable order::AtomicOrdering, syncscope::Syncscopes = SyncscopeSystem, ) - old = syncscope isa typeof(SyncscopeSystem) ? + old = + syncscope isa typeof(SyncscopeSystem) ? llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : - llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order), syncscope) + llvm_atomic_op( + $(Val(llvmop)), + ptr, + x, + llvm_from_julia_ordering(order), + syncscope, + ) return old => $op(old, x) end end diff --git a/src/internal.jl b/src/internal.jl index 3f18bc4..989ffed 100644 --- a/src/internal.jl +++ b/src/internal.jl @@ -21,10 +21,13 @@ mapop(::typeof(UnsafeAtomics.right)) = right atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}()) @inline UnsafeAtomics.modify!( - ptr::LLVMPtr, op::OP, x, order::Ordering, syncscope::Syncscopes = SyncscopeSystem, -) where {OP <: Union{typeof(+), typeof(-)}} = atomic_pointermodify( - ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope, -) + ptr::LLVMPtr, + op::OP, + x, + order::Ordering, + syncscope::Syncscopes = SyncscopeSystem, +) where {OP<:Union{typeof(+),typeof(-)}} = + atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope) @inline UnsafeAtomics.cas!( ptr::LLVMPtr, diff --git a/test/runtests.jl b/test/runtests.jl index 8df89e8..09ee4fd 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -29,18 +29,6 @@ function check_default_ordering(T::Type) xs[1] = x1 @test rmw(ptr, x2) === x1 @test xs[1] === op(x1, x2) - - # Test syncscopes. - if (op == +) || (op == -) - @info "!!!!!!!!!!!!!!!!!!!" - xs[1] = x1 - @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomicsLLVM.Internal.SyncscopeSystem) === (x1 => op(x1, x2)) - @test xs[1] === op(x1, x2) - - xs[1] = x1 - @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, UnsafeAtomicsLLVM.Internal.SyncscopeSingleThread) === (x1 => op(x1, x2)) - @test xs[1] === op(x1, x2) - end end end end @@ -69,6 +57,29 @@ function test_explicit_ordering(T::Type = UInt) xs[1] = x1 @test rmw(ptr, x2, acquire) === x1 @test xs[1] === op(x1, x2) + + # Test syncscopes. + if (op == +) || (op == -) + xs[1] = x1 + @test UnsafeAtomics.modify!( + ptr, + op, + x2, + seq_cst, + UnsafeAtomicsLLVM.Internal.SyncscopeSystem, + ) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + + xs[1] = x1 + @test UnsafeAtomics.modify!( + ptr, + op, + x2, + seq_cst, + UnsafeAtomicsLLVM.Internal.SyncscopeSingleThread, + ) === (x1 => op(x1, x2)) + @test xs[1] === op(x1, x2) + end end end end @@ -76,7 +87,6 @@ end @testset "UnsafeAtomicsLLVM" begin @testset for T in inttypes check_default_ordering(T) - # test_explicit_ordering(T) - break + test_explicit_ordering(T) end end From 3507be09850b3fd68b0527669f4d0ca7d424c3b9 Mon Sep 17 00:00:00 2001 From: Anton Smirnov Date: Tue, 6 Aug 2024 18:36:06 +0300 Subject: [PATCH 3/6] Use raw --- src/atomics.jl | 18 +++++------------- src/internal.jl | 4 ++-- test/runtests.jl | 4 ++-- 3 files changed, 9 insertions(+), 17 deletions(-) diff --git a/src/atomics.jl b/src/atomics.jl index fd48f71..8f66320 100644 --- a/src/atomics.jl +++ b/src/atomics.jl @@ -256,14 +256,6 @@ end const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} -# Syncscopes. -const SyncscopeSystem = Val{:system}() -const SyncscopeSingleThread = Val{:singlethread}() -const SyncscopeAgent = Val{:agent}() - -const Syncscopes = - Union{typeof(SyncscopeSystem),typeof(SyncscopeSingleThread),typeof(SyncscopeAgent)} - # LLVM API accepts string literal as a syncscope argument. @inline syncscope_to_string(::Type{Val{S}}) where {S} = string(S) @@ -309,8 +301,8 @@ end ptr::LLVMPtr{T,A}, val::T, order::LLVMOrderingVal, - syncscope::Syncscopes, -) where {T,A} + syncscope::Val{S}, +) where {T,A,S} @dispose ctx = Context() begin T_val = convert(LLVMType, T) T_ptr = convert(LLVMType, ptr) @@ -404,10 +396,10 @@ for (opname, op, llvmop) in binoptable ::$(typeof(op)), x::$T, order::AtomicOrdering, - syncscope::Syncscopes = SyncscopeSystem, - ) + syncscope::Val{S} = Val{:system}(), + ) where {S} old = - syncscope isa typeof(SyncscopeSystem) ? + syncscope isa Val{:system} ? llvm_atomic_op($(Val(llvmop)), ptr, x, llvm_from_julia_ordering(order)) : llvm_atomic_op( $(Val(llvmop)), diff --git a/src/internal.jl b/src/internal.jl index 989ffed..653abf1 100644 --- a/src/internal.jl +++ b/src/internal.jl @@ -25,8 +25,8 @@ mapop(::typeof(UnsafeAtomics.right)) = right op::OP, x, order::Ordering, - syncscope::Syncscopes = SyncscopeSystem, -) where {OP<:Union{typeof(+),typeof(-)}} = + syncscope::Val{S} = Val{:system}(), +) where {OP<:Union{typeof(+),typeof(-)}, S} = atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope) @inline UnsafeAtomics.cas!( diff --git a/test/runtests.jl b/test/runtests.jl index 09ee4fd..0a38e6c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,7 +66,7 @@ function test_explicit_ordering(T::Type = UInt) op, x2, seq_cst, - UnsafeAtomicsLLVM.Internal.SyncscopeSystem, + Val{:system}(), ) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) @@ -76,7 +76,7 @@ function test_explicit_ordering(T::Type = UInt) op, x2, seq_cst, - UnsafeAtomicsLLVM.Internal.SyncscopeSingleThread, + Val{:singlethread}(), ) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) end From 800952f27c77bbd3abf25ce63aa59b38eb1ebc6b Mon Sep 17 00:00:00 2001 From: Anton Smirnov Date: Tue, 6 Aug 2024 21:16:34 +0300 Subject: [PATCH 4/6] Minor refactoring --- src/atomics.jl | 6 ++---- src/internal.jl | 2 +- test/runtests.jl | 4 ++-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/atomics.jl b/src/atomics.jl index 8f66320..9b1e504 100644 --- a/src/atomics.jl +++ b/src/atomics.jl @@ -248,11 +248,9 @@ const binoptable = [ (:umin, min, LLVM.API.LLVMAtomicRMWBinOpUMin), (:fadd, +, LLVM.API.LLVMAtomicRMWBinOpFAdd), (:fsub, -, LLVM.API.LLVMAtomicRMWBinOpFSub), + (:fmax, max, LLVM.API.LLVMAtomicRMWBinOpFMax), + (:fmin, min, LLVM.API.LLVMAtomicRMWBinOpFMin), ] -if VERSION ≥ v"1.10-" - push!(binoptable, (:fmax, max, LLVM.API.LLVMAtomicRMWBinOpFMax)) - push!(binoptable, (:fmin, min, LLVM.API.LLVMAtomicRMWBinOpFMin)) -end const AtomicRMWBinOpVal = Union{(Val{binop} for (_, _, binop) in binoptable)...} diff --git a/src/internal.jl b/src/internal.jl index 653abf1..394c2b9 100644 --- a/src/internal.jl +++ b/src/internal.jl @@ -25,7 +25,7 @@ mapop(::typeof(UnsafeAtomics.right)) = right op::OP, x, order::Ordering, - syncscope::Val{S} = Val{:system}(), + syncscope::Val{S} = Val(:system), ) where {OP<:Union{typeof(+),typeof(-)}, S} = atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope) diff --git a/test/runtests.jl b/test/runtests.jl index 0a38e6c..f890d36 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -66,7 +66,7 @@ function test_explicit_ordering(T::Type = UInt) op, x2, seq_cst, - Val{:system}(), + Val(:system), ) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) @@ -76,7 +76,7 @@ function test_explicit_ordering(T::Type = UInt) op, x2, seq_cst, - Val{:singlethread}(), + Val(:singlethread), ) === (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) end From dbf6d95f5a38ac50b78a353b65bc730da5004a7c Mon Sep 17 00:00:00 2001 From: Anton Smirnov Date: Tue, 6 Aug 2024 21:22:26 +0300 Subject: [PATCH 5/6] Refactor --- src/internal.jl | 2 +- test/runtests.jl | 18 ++++-------------- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/internal.jl b/src/internal.jl index 394c2b9..e4ed556 100644 --- a/src/internal.jl +++ b/src/internal.jl @@ -26,7 +26,7 @@ mapop(::typeof(UnsafeAtomics.right)) = right x, order::Ordering, syncscope::Val{S} = Val(:system), -) where {OP<:Union{typeof(+),typeof(-)}, S} = +) where {OP<:Union{typeof(+),typeof(-)},S} = atomic_pointermodify(ptr, mapop(op), x, Val{julia_ordering_name(order)}(), syncscope) @inline UnsafeAtomics.cas!( diff --git a/test/runtests.jl b/test/runtests.jl index f890d36..d507647 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -61,23 +61,13 @@ function test_explicit_ordering(T::Type = UInt) # Test syncscopes. if (op == +) || (op == -) xs[1] = x1 - @test UnsafeAtomics.modify!( - ptr, - op, - x2, - seq_cst, - Val(:system), - ) === (x1 => op(x1, x2)) + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:system)) === + (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) xs[1] = x1 - @test UnsafeAtomics.modify!( - ptr, - op, - x2, - seq_cst, - Val(:singlethread), - ) === (x1 => op(x1, x2)) + @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:singlethread)) === + (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) end end From fd77b7bf0e13d4d84819d6889546aa28bf7ce564 Mon Sep 17 00:00:00 2001 From: Anton Smirnov Date: Tue, 6 Aug 2024 21:24:43 +0300 Subject: [PATCH 6/6] Refactor --- test/runtests.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index d507647..9ee4bfe 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -62,12 +62,12 @@ function test_explicit_ordering(T::Type = UInt) if (op == +) || (op == -) xs[1] = x1 @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:system)) === - (x1 => op(x1, x2)) + (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) xs[1] = x1 @test UnsafeAtomics.modify!(ptr, op, x2, seq_cst, Val(:singlethread)) === - (x1 => op(x1, x2)) + (x1 => op(x1, x2)) @test xs[1] === op(x1, x2) end end