[FAB-5153] Relax gossip send buffer behavior

In gossip there are send and receive buffers that are
allocated for each connection.
When the throughput of messages is too high and the send buffer
overflows, the connection to the peer is closed and the peer is removed from the membership.

From performance evaluations I did - I conclude that:
- Increasing the send buffer size would benefit to withstand
  intense bursts of messages, even when the receive buffer stays the same.
- Not closing the connection to the peer (and not removing it from the membership)
  that its corresponding send buffer overflowed helps the throughput by giving the
  runtime an opportunity to recover in spite of an intensive burst.

Change-Id: I7bc84092e366b75b6cbcaee1ea9d5320274dfc1c
Signed-off-by: yacovm <[email protected]>

gossip/comm/comm_impl.go

@@ -45,11 +45,8 @@ const (
 	defConnTimeout  = time.Second * time.Duration(2)
 	defRecvBuffSize = 20
 	defSendBuffSize = 20
-	sendOverflowErr = "Send buffer overflow"
 )
 
-var errSendOverflow = errors.New(sendOverflowErr)
-
 // SetDialTimeout sets the dial timeout
 func SetDialTimeout(timeout time.Duration) {
 	viper.Set("peer.gossip.dialTimeout", timeout)

gossip/comm/comm_test.go

@@ -195,7 +195,7 @@ func TestViperConfig(t *testing.T) {
 	assert.Equal(t, time.Duration(2)*time.Second, util.GetDurationOrDefault("peer.gossip.connTimeout", 0))
 	assert.Equal(t, time.Duration(300)*time.Millisecond, util.GetDurationOrDefault("peer.gossip.dialTimeout", 0))
 	assert.Equal(t, 20, util.GetIntOrDefault("peer.gossip.recvBuffSize", 0))
-	assert.Equal(t, 20, util.GetIntOrDefault("peer.gossip.sendBuffSize", 0))
+	assert.Equal(t, 200, util.GetIntOrDefault("peer.gossip.sendBuffSize", 0))
 }
 
 func TestHandshake(t *testing.T) {

gossip/comm/conn.go

@@ -260,7 +260,6 @@ func (conn *connection) send(msg *proto.SignedGossipMessage, onErr func(error))
 	defer conn.Unlock()
 
 	if len(conn.outBuff) == util.GetIntOrDefault("peer.gossip.sendBuffSize", defSendBuffSize) {
-		go onErr(errSendOverflow)
 		return
 	}
 

sampleconfig/core.yaml

@@ -162,7 +162,7 @@ peer:
         # Buffer size of received messages
         recvBuffSize: 20
         # Buffer size of sending messages
-        sendBuffSize: 20
+        sendBuffSize: 200
         # Time to wait before pull engine processes incoming digests (unit: second)
         digestWaitTime: 1s
         # Time to wait before pull engine removes incoming nonce (unit: second)