diff --git a/spec/std/atomic_spec.cr b/spec/std/atomic_spec.cr index 8f5fe8e8b22a..4006c33b1e1c 100644 --- a/spec/std/atomic_spec.cr +++ b/spec/std/atomic_spec.cr @@ -97,49 +97,71 @@ describe Atomic do atomic.compare_and_set([1], arr2).should eq({arr1, false}) atomic.get.should be(arr1) end + + it "explicit ordering" do + atomic = Atomic.new(1) + + atomic.compare_and_set(2, 3, :acquire, :relaxed).should eq({1, false}) + atomic.get.should eq(1) + + atomic.compare_and_set(1, 3, :acquire_release, :relaxed).should eq({1, true}) + atomic.get.should eq(3) + end end it "#adds" do atomic = Atomic.new(1) atomic.add(2).should eq(1) atomic.get.should eq(3) + atomic.add(1, :relaxed).should eq(3) + atomic.get.should eq(4) end it "#sub" do atomic = Atomic.new(1) atomic.sub(2).should eq(1) atomic.get.should eq(-1) + atomic.sub(1, :relaxed).should eq(-1) + atomic.get.should eq(-2) end it "#and" do atomic = Atomic.new(5) atomic.and(3).should eq(5) atomic.get.should eq(1) + atomic.and(7, :relaxed).should eq(1) + atomic.get.should eq(1) end it "#nand" do atomic = Atomic.new(5) atomic.nand(3).should eq(5) atomic.get.should eq(-2) + atomic.nand(7, :relaxed).should eq(-2) + atomic.get.should eq(-7) end it "#or" do atomic = Atomic.new(5) atomic.or(2).should eq(5) atomic.get.should eq(7) + atomic.or(8, :relaxed).should eq(7) + atomic.get.should eq(15) end it "#xor" do atomic = Atomic.new(5) atomic.xor(3).should eq(5) atomic.get.should eq(6) + atomic.xor(5, :relaxed).should eq(6) + atomic.get.should eq(3) end it "#max with signed" do atomic = Atomic.new(5) atomic.max(2).should eq(5) atomic.get.should eq(5) - atomic.max(10).should eq(5) + atomic.max(10, :relaxed).should eq(5) atomic.get.should eq(10) end @@ -147,7 +169,7 @@ describe Atomic do atomic = Atomic.new(5_u32) atomic.max(2_u32).should eq(5_u32) atomic.get.should eq(5_u32) - atomic.max(UInt32::MAX).should eq(5_u32) + atomic.max(UInt32::MAX, :relaxed).should eq(5_u32) atomic.get.should eq(UInt32::MAX) end @@ -165,7 +187,7 @@ describe Atomic do atomic = Atomic.new(5) atomic.min(10).should eq(5) atomic.get.should eq(5) - atomic.min(2).should eq(5) + atomic.min(2, :relaxed).should eq(5) atomic.get.should eq(2) end @@ -173,7 +195,7 @@ describe Atomic do atomic = Atomic.new(UInt32::MAX) atomic.min(10_u32).should eq(UInt32::MAX) atomic.get.should eq(10_u32) - atomic.min(15_u32).should eq(10_u32) + atomic.min(15_u32, :relaxed).should eq(10_u32) atomic.get.should eq(10_u32) end @@ -187,20 +209,28 @@ describe Atomic do atomic.get.should eq(AtomicEnum::Minus) end - it "#set" do - atomic = Atomic.new(1) - atomic.set(2).should eq(2) - atomic.get.should eq(2) - end + describe "#set" do + it "with integer" do + atomic = Atomic.new(1) + atomic.set(2).should eq(2) + atomic.get.should eq(2) + end - it "#set with nil (#4062)" do - atomic = Atomic(String?).new(nil) + it "with nil (#4062)" do + atomic = Atomic(String?).new(nil) - atomic.set("foo") - atomic.get.should eq("foo") + atomic.set("foo") + atomic.get.should eq("foo") + + atomic.set(nil) + atomic.get.should eq(nil) + end - atomic.set(nil) - atomic.get.should eq(nil) + it "explicit ordering" do + atomic = Atomic.new(1) + atomic.set(0, :release).should eq(0) + atomic.get(:acquire).should eq(0) + end end it "#lazy_set" do @@ -243,6 +273,12 @@ describe Atomic do atomic.swap(arr1).should be(arr2) atomic.get.should be(arr1) end + + it "explicit ordering" do + atomic = Atomic.new(1) + atomic.swap(2, :acquire).should eq(1) + atomic.get.should eq(2) + end end end diff --git a/src/atomic.cr b/src/atomic.cr index eedf081141d2..8569b253343a 100644 --- a/src/atomic.cr +++ b/src/atomic.cr @@ -7,6 +7,37 @@ require "llvm/enums/atomic" # reference types or `Nil`, then only `#compare_and_set`, `#swap`, `#set`, # `#lazy_set`, `#get`, and `#lazy_get` are available. struct Atomic(T) + # Specifies how memory accesses, including non atomic, are to be reordered + # around atomics. Follows the C/C++ semantics: + # . + # + # By default atomics use the sequentially consistent ordering, which has the + # strongest guarantees. If all you need is to increment a counter, a relaxed + # ordering may be enough. If you need to synchronize access to other memory + # (e.g. locks) you may try the acquire/release semantics that may be faster on + # some architectures (e.g. X86) but remember that an acquire must be paired + # with a release for the ordering to be guaranteed. + enum Ordering + Relaxed = LLVM::AtomicOrdering::Monotonic + Acquire = LLVM::AtomicOrdering::Acquire + Release = LLVM::AtomicOrdering::Release + AcquireRelease = LLVM::AtomicOrdering::AcquireRelease + SequentiallyConsistent = LLVM::AtomicOrdering::SequentiallyConsistent + end + + # Adds an explicit memory barrier with the specified memory order guarantee. + # + # Atomics on weakly-ordered CPUs (e.g. ARM32) may not guarantee memory order + # of other memory accesses, and an explicit memory barrier is thus required. + # + # Notes: + # - X86 is strongly-ordered and trying to add a fence should be a NOOP; + # - AArch64 guarantees memory order and doesn't need explicit fences in + # addition to the atomics (but may need barriers in other cases). + macro fence(ordering = :sequentially_consistent) + ::Atomic::Ops.fence({{ordering}}, false) + end + # Creates an Atomic with the given initial value. def initialize(@value : T) {% if !T.union? && (T == Char || T < Int::Primitive || T < Enum) %} @@ -38,6 +69,52 @@ struct Atomic(T) Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :sequentially_consistent, :sequentially_consistent) end + # Compares this atomic's value with *cmp* using explicit memory orderings: + # + # * if they are equal, sets the value to *new*, and returns `{old_value, true}` + # * if they are not equal the value remains the same, and returns `{old_value, false}` + # + # Reference types are compared by `#same?`, not `#==`. + # + # ``` + # atomic = Atomic.new(0_u32) + # + # value = atomic.get(:acquire) + # loop do + # value, success = atomic.compare_and_set(value, value &+ 1, :acquire_release, :acquire) + # break if success + # end + # ``` + def compare_and_set(cmp : T, new : T, success_ordering : Ordering, failure_ordering : Ordering) : {T, Bool} + case {success_ordering, failure_ordering} + when {.relaxed?, .relaxed?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :monotonic, :monotonic) + when {.acquire?, .relaxed?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :acquire, :monotonic) + when {.acquire?, .acquire?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :acquire, :acquire) + when {.release?, .relaxed?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :release, :monotonic) + when {.release?, .acquire?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :release, :acquire) + when {.acquire_release?, .relaxed?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :acquire_release, :monotonic) + when {.acquire_release?, .acquire?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :acquire_release, :acquire) + when {.sequentially_consistent?, .relaxed?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :sequentially_consistent, :monotonic) + when {.sequentially_consistent?, .acquire?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :sequentially_consistent, :acquire) + when {.sequentially_consistent?, .sequentially_consistent?} + Ops.cmpxchg(pointerof(@value), cmp.as(T), new.as(T), :sequentially_consistent, :sequentially_consistent) + else + if failure_ordering.release? || failure_ordering.acquire_release? + raise ArgumentError.new("Failure ordering cannot include release semantics") + end + raise ArgumentError.new("Failure ordering shall be no stronger than success ordering") + end + end + # Performs `atomic_value &+= value`. Returns the old value. # # `T` cannot contain any reference types. @@ -47,9 +124,9 @@ struct Atomic(T) # atomic.add(2) # => 1 # atomic.get # => 3 # ``` - def add(value : T) : T + def add(value : T, ordering : Ordering = :sequentially_consistent) : T check_reference_type - Ops.atomicrmw(:add, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:add, pointerof(@value), value, ordering) end # Performs `atomic_value &-= value`. Returns the old value. @@ -61,9 +138,9 @@ struct Atomic(T) # atomic.sub(2) # => 9 # atomic.get # => 7 # ``` - def sub(value : T) : T + def sub(value : T, ordering : Ordering = :sequentially_consistent) : T check_reference_type - Ops.atomicrmw(:sub, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:sub, pointerof(@value), value, ordering) end # Performs `atomic_value &= value`. Returns the old value. @@ -75,9 +152,9 @@ struct Atomic(T) # atomic.and(3) # => 5 # atomic.get # => 1 # ``` - def and(value : T) : T + def and(value : T, ordering : Ordering = :sequentially_consistent) : T check_reference_type - Ops.atomicrmw(:and, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:and, pointerof(@value), value, ordering) end # Performs `atomic_value = ~(atomic_value & value)`. Returns the old value. @@ -89,9 +166,9 @@ struct Atomic(T) # atomic.nand(3) # => 5 # atomic.get # => -2 # ``` - def nand(value : T) : T + def nand(value : T, ordering : Ordering = :sequentially_consistent) : T check_reference_type - Ops.atomicrmw(:nand, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:nand, pointerof(@value), value, ordering) end # Performs `atomic_value |= value`. Returns the old value. @@ -103,9 +180,9 @@ struct Atomic(T) # atomic.or(2) # => 5 # atomic.get # => 7 # ``` - def or(value : T) : T + def or(value : T, ordering : Ordering = :sequentially_consistent) : T check_reference_type - Ops.atomicrmw(:or, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:or, pointerof(@value), value, ordering) end # Performs `atomic_value ^= value`. Returns the old value. @@ -117,9 +194,9 @@ struct Atomic(T) # atomic.xor(3) # => 5 # atomic.get # => 6 # ``` - def xor(value : T) : T + def xor(value : T, ordering : Ordering = :sequentially_consistent) : T check_reference_type - Ops.atomicrmw(:xor, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:xor, pointerof(@value), value, ordering) end # Performs `atomic_value = {atomic_value, value}.max`. Returns the old value. @@ -135,18 +212,18 @@ struct Atomic(T) # atomic.max(10) # => 5 # atomic.get # => 10 # ``` - def max(value : T) + def max(value : T, ordering : Ordering = :sequentially_consistent) check_reference_type {% if T < Enum %} if @value.value.is_a?(Int::Signed) - Ops.atomicrmw(:max, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:max, pointerof(@value), value, ordering) else - Ops.atomicrmw(:umax, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:umax, pointerof(@value), value, ordering) end {% elsif T < Int::Signed %} - Ops.atomicrmw(:max, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:max, pointerof(@value), value, ordering) {% else %} - Ops.atomicrmw(:umax, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:umax, pointerof(@value), value, ordering) {% end %} end @@ -163,18 +240,18 @@ struct Atomic(T) # atomic.min(3) # => 5 # atomic.get # => 3 # ``` - def min(value : T) + def min(value : T, ordering : Ordering = :sequentially_consistent) check_reference_type {% if T < Enum %} if @value.value.is_a?(Int::Signed) - Ops.atomicrmw(:min, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:min, pointerof(@value), value, ordering) else - Ops.atomicrmw(:umin, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:umin, pointerof(@value), value, ordering) end {% elsif T < Int::Signed %} - Ops.atomicrmw(:min, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:min, pointerof(@value), value, ordering) {% else %} - Ops.atomicrmw(:umin, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:umin, pointerof(@value), value, ordering) {% end %} end @@ -185,12 +262,12 @@ struct Atomic(T) # atomic.swap(10) # => 5 # atomic.get # => 10 # ``` - def swap(value : T) + def swap(value : T, ordering : Ordering = :sequentially_consistent) {% if T.union_types.all? { |t| t == Nil || t < Reference } && T != Nil %} - address = Ops.atomicrmw(:xchg, pointerof(@value).as(LibC::SizeT*), LibC::SizeT.new(value.as(Void*).address), :sequentially_consistent, false) + address = atomicrmw(:xchg, pointerof(@value).as(LibC::SizeT*), LibC::SizeT.new(value.as(Void*).address), ordering) Pointer(T).new(address).as(T) {% else %} - Ops.atomicrmw(:xchg, pointerof(@value), value, :sequentially_consistent, false) + atomicrmw(:xchg, pointerof(@value), value, ordering) {% end %} end @@ -201,8 +278,17 @@ struct Atomic(T) # atomic.set(10) # => 10 # atomic.get # => 10 # ``` - def set(value : T) : T - Ops.store(pointerof(@value), value.as(T), :sequentially_consistent, true) + def set(value : T, ordering : Ordering = :sequentially_consistent) : T + case ordering + in .relaxed? + Ops.store(pointerof(@value), value.as(T), :monotonic, true) + in .release? + Ops.store(pointerof(@value), value.as(T), :release, true) + in .sequentially_consistent? + Ops.store(pointerof(@value), value.as(T), :sequentially_consistent, true) + in .acquire?, .acquire_release? + raise ArgumentError.new("Atomic store cannot have acquire semantic") + end value end @@ -213,15 +299,28 @@ struct Atomic(T) # atomic.lazy_set(10) # => 10 # atomic.get # => 10 # ``` + # + # NOTE: use with caution, this may break atomic guarantees. def lazy_set(@value : T) : T end # Atomically returns this atomic's value. - def get : T - Ops.load(pointerof(@value), :sequentially_consistent, true) + def get(ordering : Ordering = :sequentially_consistent) : T + case ordering + in .relaxed? + Ops.load(pointerof(@value), :monotonic, true) + in .acquire? + Ops.load(pointerof(@value), :acquire, true) + in .sequentially_consistent? + Ops.load(pointerof(@value), :sequentially_consistent, true) + in .release?, .acquire_release? + raise ArgumentError.new("Atomic load cannot have release semantic") + end end # **Non-atomically** returns this atomic's value. + # + # NOTE: use with caution, this may break atomic guarantees. def lazy_get @value end @@ -232,6 +331,21 @@ struct Atomic(T) {% end %} end + private macro atomicrmw(operation, pointer, value, ordering) + case ordering + in .relaxed? + Ops.atomicrmw({{operation}}, {{pointer}}, {{value}}, :monotonic, false) + in .acquire? + Ops.atomicrmw({{operation}}, {{pointer}}, {{value}}, :acquire, false) + in .release? + Ops.atomicrmw({{operation}}, {{pointer}}, {{value}}, :release, false) + in .acquire_release? + Ops.atomicrmw({{operation}}, {{pointer}}, {{value}}, :acquire_release, false) + in .sequentially_consistent? + Ops.atomicrmw({{operation}}, {{pointer}}, {{value}}, :sequentially_consistent, false) + end + end + # :nodoc: module Ops # Defines methods that directly map to LLVM instructions related to atomic operations. @@ -279,11 +393,18 @@ struct Atomic::Flag # Atomically tries to set the flag. Only succeeds and returns `true` if the # flag wasn't previously set; returns `false` otherwise. def test_and_set : Bool - Atomic::Ops.atomicrmw(:xchg, pointerof(@value), 1_u8, :sequentially_consistent, false) == 0_u8 + ret = Atomic::Ops.atomicrmw(:xchg, pointerof(@value), 1_u8, :sequentially_consistent, false) == 0_u8 + {% if flag?(:arm) %} + Atomic::Ops.fence(:sequentially_consistent, false) if ret + {% end %} + ret end # Atomically clears the flag. def clear : Nil + {% if flag?(:arm) %} + Atomic::Ops.fence(:sequentially_consistent, false) + {% end %} Atomic::Ops.store(pointerof(@value), 0_u8, :sequentially_consistent, true) end end diff --git a/src/compiler/crystal/codegen/primitives.cr b/src/compiler/crystal/codegen/primitives.cr index 80ba7d832a41..eb048076de50 100644 --- a/src/compiler/crystal/codegen/primitives.cr +++ b/src/compiler/crystal/codegen/primitives.cr @@ -1165,8 +1165,11 @@ class Crystal::CodeGenVisitor call = check_atomic_call(call, target_def) ptr, cmp, new, success_ordering, failure_ordering = call_args - success_ordering = atomic_ordering_get_const(call.args[-2], success_ordering) - failure_ordering = atomic_ordering_get_const(call.args[-1], failure_ordering) + success_node = call.args[-2] + failure_node = call.args[-1] + success_ordering = atomic_ordering_get_const(success_node, success_ordering) + failure_ordering = atomic_ordering_get_const(failure_node, failure_ordering) + validate_atomic_cmpxchg_ordering(success_node, success_ordering, failure_node, failure_ordering) value = builder.cmpxchg(ptr, cmp, new, success_ordering, failure_ordering) value_type = node.type.as(TupleInstanceType) @@ -1192,9 +1195,14 @@ class Crystal::CodeGenVisitor call = check_atomic_call(call, target_def) ordering, _ = call_args - ordering = atomic_ordering_get_const(call.args[0], ordering) + ordering_node = call.args[0] + ordering = atomic_ordering_get_const(ordering_node, ordering) singlethread = bool_from_bool_literal(call.args[1]) + ordering_node.raise "must be atomic" if ordering.not_atomic? + ordering_node.raise "cannot be unordered" if ordering.unordered? + ordering_node.raise "must have acquire, release, acquire_release or sequentially_consistent ordering" if ordering.monotonic? + builder.fence(ordering, singlethread) llvm_nil end @@ -1288,6 +1296,23 @@ class Crystal::CodeGenVisitor end end + def validate_atomic_cmpxchg_ordering(success_node, success_ordering, failure_node, failure_ordering) + success_node.raise "must be atomic" if success_ordering.not_atomic? + success_node.raise "cannot be unordered" if success_ordering.unordered? + + failure_node.raise "must be atomic" if failure_ordering.not_atomic? + failure_node.raise "cannot be unordered" if failure_ordering.unordered? + failure_node.raise "cannot include release semantics" if failure_ordering.release? || failure_ordering.acquire_release? + + {% if LibLLVM::IS_LT_130 %} + # Atomic(T) macros enforce this rule to provide a consistent public API + # regardless of which LLVM version crystal was compiled with. The compiler, + # however, only needs to make sure that the codegen is correct for the LLVM + # version + failure_node.raise "shall be no stronger than success ordering" if failure_ordering > success_ordering + {% end %} + end + def bool_from_bool_literal(node) unless node.is_a?(BoolLiteral) node.raise "BUG: expected bool literal" diff --git a/src/crystal/rw_lock.cr b/src/crystal/rw_lock.cr index de61fd31be14..a47472a99407 100644 --- a/src/crystal/rw_lock.cr +++ b/src/crystal/rw_lock.cr @@ -1,37 +1,55 @@ # :nodoc: class Crystal::RWLock - @writer = Atomic(Int32).new(0) + private UNLOCKED = 0 + private LOCKED = 1 + + @writer = Atomic(Int32).new(UNLOCKED) @readers = Atomic(Int32).new(0) - def read_lock + def read_lock : Nil loop do - while @writer.get != 0 + while @writer.get(:relaxed) != UNLOCKED Intrinsics.pause end - @readers.add(1) + @readers.add(1, :acquire) - break if @writer.get == 0 + if @writer.get(:acquire) == UNLOCKED + {% if flag?(:arm) %} + Atomic.fence(:acquire) + {% end %} + return + end - @readers.sub(1) + @readers.sub(1, :release) end end - def read_unlock - @readers.sub(1) + def read_unlock : Nil + {% if flag?(:arm) %} + Atomic.fence(:release) + {% end %} + @readers.sub(1, :release) end - def write_lock - while @writer.swap(1) != 0 + def write_lock : Nil + while @writer.swap(LOCKED, :acquire) != UNLOCKED Intrinsics.pause end - while @readers.get != 0 + while @readers.get(:acquire) != 0 Intrinsics.pause end + + {% if flag?(:arm) %} + Atomic.fence(:acquire) + {% end %} end - def write_unlock - @writer.lazy_set(0) + def write_unlock : Nil + {% if flag?(:arm) %} + Atomic.fence(:release) + {% end %} + @writer.set(UNLOCKED, :release) end end diff --git a/src/crystal/spin_lock.cr b/src/crystal/spin_lock.cr index e3136cdf3fe9..0792ce652159 100644 --- a/src/crystal/spin_lock.cr +++ b/src/crystal/spin_lock.cr @@ -1,28 +1,31 @@ # :nodoc: class Crystal::SpinLock + private UNLOCKED = 0 + private LOCKED = 1 + {% if flag?(:preview_mt) %} - @m = Atomic(Int32).new(0) + @m = Atomic(Int32).new(UNLOCKED) {% end %} def lock {% if flag?(:preview_mt) %} - while @m.swap(1) == 1 - while @m.get == 1 + while @m.swap(LOCKED, :acquire) == LOCKED + while @m.get(:relaxed) == LOCKED Intrinsics.pause end end - {% if flag?(:aarch64) %} - Atomic::Ops.fence :sequentially_consistent, false - {% end %} + {% if flag?(:arm) %} + Atomic.fence(:acquire) + {% end %} {% end %} end def unlock {% if flag?(:preview_mt) %} - {% if flag?(:aarch64) %} - Atomic::Ops.fence :sequentially_consistent, false + {% if flag?(:arm) %} + Atomic.fence(:release) {% end %} - @m.lazy_set(0) + @m.set(UNLOCKED, :release) {% end %} end diff --git a/src/mutex.cr b/src/mutex.cr index 83dd4bb53429..38d34c14b070 100644 --- a/src/mutex.cr +++ b/src/mutex.cr @@ -16,7 +16,10 @@ require "crystal/spin_lock" # mutex from the same fiber will deadlock. Any fiber can unlock the mutex, even # if it wasn't previously locked. class Mutex - @state = Atomic(Int32).new(0) + private UNLOCKED = 0 + private LOCKED = 1 + + @state = Atomic(Int32).new(UNLOCKED) @mutex_fiber : Fiber? @lock_count = 0 @queue = Deque(Fiber).new @@ -34,7 +37,10 @@ class Mutex @[AlwaysInline] def lock : Nil - if @state.swap(1) == 0 + if @state.swap(LOCKED, :acquire) == UNLOCKED + {% if flag?(:arm) %} + Atomic.fence(:acquire) + {% end %} @mutex_fiber = Fiber.current unless @protection.unchecked? return end @@ -49,20 +55,24 @@ class Mutex end lock_slow - nil end @[NoInline] - private def lock_slow + private def lock_slow : Nil loop do break if try_lock @lock.sync do @queue_count.add(1) - if @state.get == 0 - if @state.swap(1) == 0 - @queue_count.add(-1) - @mutex_fiber = Fiber.current + + if @state.get(:relaxed) == UNLOCKED + if @state.swap(LOCKED, :acquire) == UNLOCKED + {% if flag?(:arm) %} + Atomic.fence(:acquire) + {% end %} + @queue_count.sub(1) + + @mutex_fiber = Fiber.current unless @protection.unchecked? return end end @@ -72,20 +82,21 @@ class Mutex Crystal::Scheduler.reschedule end - @mutex_fiber = Fiber.current - nil + @mutex_fiber = Fiber.current unless @protection.unchecked? end private def try_lock i = 1000 - while @state.swap(1) != 0 - while @state.get != 0 + while @state.swap(LOCKED, :acquire) != UNLOCKED + while @state.get(:relaxed) != UNLOCKED Intrinsics.pause i &-= 1 return false if i == 0 end end - + {% if flag?(:arm) %} + Atomic.fence(:acquire) + {% end %} true end @@ -107,10 +118,7 @@ class Mutex @mutex_fiber = nil end - {% if flag?(:aarch64) %} - Atomic::Ops.fence :sequentially_consistent, false - {% end %} - @state.lazy_set(0) + @state.set(UNLOCKED, :release) if @queue_count.get == 0 return @@ -127,8 +135,6 @@ class Mutex end end fiber.enqueue if fiber - - nil end def synchronize(&)