Retry building a message when sending results in wrong sequence

relayer/provider/cosmos/provider.go

@@ -1578,16 +1578,75 @@ func (cc *CosmosProvider) SendMessage(ctx context.Context, msg provider.RelayerM
 // of that transaction will be logged. A boolean indicating if a transaction was successfully
 // sent and executed successfully is returned.
 func (cc *CosmosProvider) SendMessages(ctx context.Context, msgs []provider.RelayerMessage) (*provider.RelayerTxResponse, bool, error) {
-	var (
-		txb     client.TxBuilder
-		txBytes []byte
-		res     *sdk.TxResponse
-	)
+	var res *sdk.TxResponse
+
+	if err := retry.Do(func() error {
+		txBytes, err := cc.buildMessages(ctx, msgs)
+		if err != nil {
+			return err
+		}
+
+		res, err = cc.BroadcastTx(ctx, txBytes)
+		if err != nil {
+			if err == sdkerrors.ErrWrongSequence {
+				// Allow retrying if we got an invalid sequence error when attempting to broadcast this tx.
+				return err
+			}
+
+			// Don't retry if BroadcastTx resulted in any other error.
+			// (This was the previous behavior. Unclear if that is still desired.)
+			return retry.Unrecoverable(err)
+		}
+
+		return nil
+	}, retry.Context(ctx), RtyAtt, RtyDel, RtyErr, retry.OnRetry(func(n uint, err error) {
+		cc.log.Info(
+			"Error building or broadcasting transaction",
+			zap.String("chain_id", cc.PCfg.ChainID),
+			zap.Uint("attempt", n+1),
+			zap.Uint("max_attempts", RtyAttNum),
+			zap.Error(err),
+		)
+	})); err != nil || res == nil {
+		return nil, false, err
+	}
+
+	// Parse events and build a map where the key is event.Type+"."+attribute.Key
+	events := make(map[string]string, 1)
+	for _, logs := range res.Logs {
+		for _, ev := range logs.Events {
+			for _, attr := range ev.Attributes {
+				key := ev.Type + "." + attr.Key
+				events[key] = attr.Value
+			}
+		}
+	}
+
+	rlyRes := &provider.RelayerTxResponse{
+		Height: res.Height,
+		TxHash: res.TxHash,
+		Code:   res.Code,
+		Data:   res.Data,
+		Events: events,
+	}
+
+	// transaction was executed, log the success or failure using the tx response code
+	// NOTE: error is nil, logic should use the returned error to determine if the
+	// transaction was successfully executed.
+	if rlyRes.Code != 0 {
+		cc.LogFailedTx(rlyRes, nil, msgs)
+		return rlyRes, false, fmt.Errorf("transaction failed with code: %d", res.Code)
+	}
+
+	cc.LogSuccessTx(res, msgs)
+	return rlyRes, true, nil
+}
 
+func (cc *CosmosProvider) buildMessages(ctx context.Context, msgs []provider.RelayerMessage) ([]byte, error) {
 	// Query account details
 	txf, err := cc.PrepareFactory(cc.TxFactory())
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	// TODO: Make this work with new CalculateGas method
@@ -1596,21 +1655,22 @@ func (cc *CosmosProvider) SendMessages(ctx context.Context, msgs []provider.Rela
 	// If users pass gas adjustment, then calculate gas
 	_, adjusted, err := cc.CalculateGas(ctx, txf, CosmosMsgs(msgs...)...)
 	if err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	// Set the gas amount on the transaction factory
 	txf = txf.WithGas(adjusted)
 
+	var txb client.TxBuilder
 	// Build the transaction builder & retry on failures
-	if err = retry.Do(func() error {
+	if err := retry.Do(func() error {
 		txb, err = tx.BuildUnsignedTx(txf, CosmosMsgs(msgs...)...)
 		if err != nil {
 			return err
 		}
-		return err
+		return nil
 	}, retry.Context(ctx), RtyAtt, RtyDel, RtyErr); err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	// Attach the signature to the transaction
@@ -1621,60 +1681,29 @@ func (cc *CosmosProvider) SendMessages(ctx context.Context, msgs []provider.Rela
 
 	done := cc.SetSDKContext()
 
-	if err = retry.Do(func() error {
-		if err = tx.Sign(txf, cc.PCfg.Key, txb, false); err != nil {
+	if err := retry.Do(func() error {
+		if err := tx.Sign(txf, cc.PCfg.Key, txb, false); err != nil {
 			return err
 		}
-		return err
+		return nil
 	}, retry.Context(ctx), RtyAtt, RtyDel, RtyErr); err != nil {
-		return nil, false, err
+		return nil, err
 	}
 
 	done()
 
+	var txBytes []byte
 	// Generate the transaction bytes
-	if err = retry.Do(func() error {
+	if err := retry.Do(func() error {
+		var err error
 		txBytes, err = cc.Codec.TxConfig.TxEncoder()(txb.GetTx())
 		if err != nil {
 			return err
 		}
-		return err
+		return nil
 	}, retry.Context(ctx), RtyAtt, RtyDel, RtyErr); err != nil {
-		return nil, false, err
-	}
-
-	res, err = cc.BroadcastTx(ctx, txBytes)
-	if err != nil || res == nil {
-		return nil, false, err
-	}
-
-	// Parse events and build a map where the key is event.Type+"."+attribute.Key
-	events := make(map[string]string, 1)
-	for _, logs := range res.Logs {
-		for _, ev := range logs.Events {
-			for _, attr := range ev.Attributes {
-				key := ev.Type + "." + attr.Key
-				events[key] = attr.Value
-			}
-		}
-	}
-
-	rlyRes := &provider.RelayerTxResponse{
-		Height: res.Height,
-		TxHash: res.TxHash,
-		Code:   res.Code,
-		Data:   res.Data,
-		Events: events,
-	}
-
-	// transaction was executed, log the success or failure using the tx response code
-	// NOTE: error is nil, logic should use the returned error to determine if the
-	// transaction was successfully executed.
-	if rlyRes.Code != 0 {
-		cc.LogFailedTx(rlyRes, err, msgs)
-		return rlyRes, false, fmt.Errorf("transaction failed with code: %d", res.Code)
+		return nil, err
 	}
 
-	cc.LogSuccessTx(res, msgs)
-	return rlyRes, true, nil
+	return txBytes, nil
 }