[FAB-5633] Add logging to deliverclient/client.go

The deliveryservice/client.go stores retry logic that
plays a role when the ordering service node is unreachable or malfunctioning.

It lacks logging, and thus makes it hard to analyze
postmortem and live situations.

Change-Id: I7f3a3d52c5fc67bed400f0007aebdb5e61419ec9
Signed-off-by: yacovm <[email protected]>
(cherry picked from commit e1f7c50)
Signed-off-by: Gari Singh <[email protected]>

core/deliverservice/client.go

@@ -98,8 +98,10 @@ func (bc *broadcastClient) try(action func() (interface{}, error)) (interface{},
 		if err != nil {
 			backoffDuration, retry = bc.shouldRetry(attempt, time.Since(start))
 			if !retry {
+				logger.Warning("Got error:", err, "at", attempt, "attempt. Ceasing to retry")
 				break
 			}
+			logger.Warning("Got error:", err, ",at", attempt, "attempt. Retrying in", backoffDuration)
 			bc.sleep(backoffDuration)
 			continue
 		}
@@ -135,11 +137,13 @@ func (bc *broadcastClient) sleep(duration time.Duration) {
 
 func (bc *broadcastClient) connect() error {
 	conn, endpoint, err := bc.prod.NewConnection()
+	logger.Debug("Connected to", endpoint)
 	if err != nil {
 		logger.Error("Failed obtaining connection:", err)
 		return err
 	}
 	ctx, cf := context.WithCancel(context.Background())
+	logger.Debug("Establishing gRPC stream with", endpoint, "...")
 	abc, err := bc.createClient(conn).Deliver(ctx)
 	if err != nil {
 		logger.Error("Connection to ", endpoint, "established but was unable to create gRPC stream:", err)
@@ -150,13 +154,16 @@ func (bc *broadcastClient) connect() error {
 	if err == nil {
 		return nil
 	}
+	logger.Warning("Failed running post-connection procedures:", err)
 	// If we reached here, lets make sure connection is closed
 	// and nullified before we return
 	bc.Disconnect()
 	return err
 }
 
 func (bc *broadcastClient) afterConnect(conn *grpc.ClientConn, abc orderer.AtomicBroadcast_DeliverClient, cf context.CancelFunc) error {
+	logger.Debug("Entering")
+	defer logger.Debug("Exiting")
 	bc.Lock()
 	bc.conn = &connection{ClientConn: conn, cancel: cf}
 	bc.BlocksDeliverer = abc
@@ -194,6 +201,8 @@ func (bc *broadcastClient) shouldStop() bool {
 
 // Close makes the client close its connection and shut down
 func (bc *broadcastClient) Close() {
+	logger.Debug("Entering")
+	defer logger.Debug("Exiting")
 	bc.Lock()
 	defer bc.Unlock()
 	if bc.shouldStop() {
@@ -209,6 +218,8 @@ func (bc *broadcastClient) Close() {
 
 // Disconnect makes the client close the existing connection
 func (bc *broadcastClient) Disconnect() {
+	logger.Debug("Entering")
+	defer logger.Debug("Exiting")
 	bc.Lock()
 	defer bc.Unlock()
 	if bc.conn == nil {