From d7c49ccd3496f90b18e650393edaf43df8e26a19 Mon Sep 17 00:00:00 2001 From: Sugu Sougoumarane Date: Fri, 29 Sep 2017 22:21:42 -0700 Subject: [PATCH] messages: fix deadlock BUG=67060474 The following deadlock is possible: 1. runSend obtains a lock. 2. Poller gets trigerred, and waits for lock. 3. runSend calls this poller trigger, but the trigger will hang because it cannot return until poller returns. Unfortunately, this race is not reproduceable in a test because it's a CPU vs. CPU race. So, we just have to rely on reasoning. --- go/vt/vttablet/tabletserver/messager/message_manager.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/go/vt/vttablet/tabletserver/messager/message_manager.go b/go/vt/vttablet/tabletserver/messager/message_manager.go index a91ace8e018..027c050e92a 100644 --- a/go/vt/vttablet/tabletserver/messager/message_manager.go +++ b/go/vt/vttablet/tabletserver/messager/message_manager.go @@ -344,7 +344,13 @@ func (mm *messageManager) runSend() { } if mm.messagesPending { // Trigger the poller to fetch more. - mm.pollerTicks.Trigger() + // Do this as a separate goroutine. Otherwise, this could cause + // the following deadlock: + // 1. runSend obtains a lock + // 2. Poller gets trigerred, and waits for lock. + // 3. runSend calls this function, but the trigger will hang because + // this function cannot return until poller returns. + go mm.pollerTicks.Trigger() } mm.cond.Wait() }