SpinLock causes deadlock :/

stdlib/Distributed/src/cluster.jl

@@ -95,7 +95,7 @@ end
 @enum WorkerState W_CREATED W_CONNECTED W_TERMINATING W_TERMINATED
 mutable struct Worker
     id::Int
-    msg_lock::Threads.SpinLock # Lock for del_msgs, add_msgs, and gcflag, SpinLock since it needs to be used from finalizers
+    msg_lock::Threads.ReentrantLock # Lock for del_msgs, add_msgs, and gcflag
     del_msgs::Array{Any,1}
     add_msgs::Array{Any,1}
     gcflag::Bool
@@ -134,7 +134,7 @@ mutable struct Worker
         if haskey(map_pid_wrkr, id)
             return map_pid_wrkr[id]
         end
-        w=new(id, Threads.SpinLock(), [], [], false, W_CREATED, Threads.Condition(), time(), conn_func)
+        w=new(id, Threads.ReentrantLock(), [], [], false, W_CREATED, Threads.Condition(), time(), conn_func)
         w.initialized = Event()
         register_worker(w)
         w

stdlib/Distributed/src/remotecall.jl

@@ -247,13 +247,18 @@ function del_clients(pairs::Vector)
     end
 end
 
+# The task below is coalescing the `flush_gc_msgs` call
+# across multiple producers, see `send_del_client`,
+# and `send_add_client`.
+# XXX: Is this worth the additional complexity?
+#      `flush_gc_msgs` has to iterate over all connected workers.
 const any_gc_flag = Threads.Condition()
 function start_gc_msgs_task()
-    @async begin
+    Threads.@spawn begin
         while true
             lock(any_gc_flag) do
                 wait(any_gc_flag)
-                flush_gc_msgs()
+                flush_gc_msgs() # handle's throws internally
             end
         end
     end
@@ -266,13 +271,15 @@ function send_del_client(rr)
     elseif id_in_procs(rr.where) # process only if a valid worker
         w = worker_from_id(rr.where)
         msg = (remoteref_id(rr), myid())
-        # Lock is SpinLock an can thus be acquired from finalizer
-        lock(w.msg_lock) do
-          push!(w.del_msgs, msg)
-          w.gcflag = true
-        end
-        lock(any_gc_flag) do
-            notify(any_gc_flag)
+        # We cannot acquire locks from finalizers
+        @async begin
+            lock(w.msg_lock) do
+              push!(w.del_msgs, msg)
+              w.gcflag = true
+            end
+            lock(any_gc_flag) do
+                notify(any_gc_flag)
+            end
         end
     end
 end