guard and notary retry

agents/agents/guard/guard.go

@@ -178,6 +178,11 @@ func NewGuard(ctx context.Context, cfg config.AgentConfig, omniRPCClient omnirpc
 	guard.originLatestStates = make(map[uint32]types.State, len(guard.domains))
 	guard.handler = handler
 	guard.txSubmitter = submitter.NewTransactionSubmitter(handler, guard.unbondedSigner, omniRPCClient, guardDB.SubmitterDB(), &cfg.SubmitterConfig)
+
+	if cfg.MaxRetrySeconds == 0 {
+		cfg.MaxRetrySeconds = 30
+	}
+
 	guard.retryConfig = []retry.WithBackoffConfigurator{
 		retry.WithMaxAttemptTime(time.Second * time.Duration(cfg.MaxRetrySeconds)),
 	}
@@ -276,16 +281,19 @@ func (g Guard) loadSummitLatestStates(parentCtx context.Context) {
 		))
 
 		originID := domain.Config().DomainID
-		latestState, err := g.domains[g.summitDomainID].Summit().GetLatestAgentState(ctx, originID, g.bondedSigner)
-		if err != nil {
-			latestState = nil
-			logger.Errorf("Failed calling GetLatestAgentState for originID %d on the Summit contract: err = %v", originID, err)
-			span.AddEvent("Failed calling GetLatestAgentState for originID on the Summit contract", trace.WithAttributes(
-				attribute.Int("originID", int(originID)),
-				attribute.String("err", err.Error()),
-			))
+
+		var latestState types.State
+		var err error
+		contractCall := func(ctx context.Context) error {
+			latestState, err = g.domains[g.summitDomainID].Summit().GetLatestAgentState(ctx, originID, g.bondedSigner)
+			if err != nil {
+				return fmt.Errorf("failed calling GetLatestAgentState for originID %d on the Summit contract: err = %w", originID, err)
+			}
+
+			return nil
 		}
-		if latestState != nil && latestState.Nonce() > uint32(0) {
+		err = retry.WithBackoff(ctx, contractCall, g.retryConfig...)
+		if err == nil && latestState.Nonce() > uint32(0) {
 			g.summitLatestStates[originID] = latestState
 		}
 
@@ -295,12 +303,37 @@ func (g Guard) loadSummitLatestStates(parentCtx context.Context) {
 
 //nolint:cyclop
 func (g Guard) loadOriginLatestStates(parentCtx context.Context) {
-	for _, domain := range g.domains {
+	for _, d := range g.domains {
+		domain := d
 		ctx, span := g.handler.Tracer().Start(parentCtx, "loadOriginLatestStates", trace.WithAttributes(
 			attribute.Int("domain", int(domain.Config().DomainID)),
 		))
 
 		originID := domain.Config().DomainID
+
+		// var latestState types.State
+		// var err error
+		// contractCall := func(ctx context.Context) error {
+		// 	latestState, err = domain.Origin().SuggestLatestState(ctx)
+		// 	if err != nil {
+		// 		latestState = nil
+		// 		return fmt.Errorf("failed calling SuggestLatestState for originID %d on the Origin contract: %w", originID, err)
+		// 	}
+		//
+		// 	return nil
+		// }
+		// err = retry.WithBackoff(ctx, contractCall, g.retryConfig...)
+		// if latestState == nil || latestState.Nonce() == uint32(0) {
+		// 	logger.Errorf("No latest state found for origin id %d", originID)
+		// 	span.AddEvent("No latest state found for origin id", trace.WithAttributes(
+		// 		attribute.Int("originID", int(originID)),
+		// 	))
+		// }
+		//
+		// if err == nil {
+		// 	g.originLatestStates[originID] = latestState
+		// }
+
 		latestState, err := domain.Origin().SuggestLatestState(ctx)
 		if err != nil {
 			latestState = nil

agents/agents/notary/notary.go

@@ -7,22 +7,22 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
+	"github.com/ethereum/go-ethereum/common"
 	ethTypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/synapsecns/sanguine/agents/agents/notary/db"
-	"github.com/synapsecns/sanguine/core/metrics"
-	signerConfig "github.com/synapsecns/sanguine/ethergo/signer/config"
-	"github.com/synapsecns/sanguine/ethergo/submitter"
-	omnirpcClient "github.com/synapsecns/sanguine/services/omnirpc/client"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/trace"
-
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/synapsecns/sanguine/agents/config"
 	"github.com/synapsecns/sanguine/agents/contracts/summit"
 	"github.com/synapsecns/sanguine/agents/domains"
 	"github.com/synapsecns/sanguine/agents/domains/evm"
 	"github.com/synapsecns/sanguine/agents/types"
+	"github.com/synapsecns/sanguine/core/metrics"
+	"github.com/synapsecns/sanguine/core/retry"
+	signerConfig "github.com/synapsecns/sanguine/ethergo/signer/config"
 	"github.com/synapsecns/sanguine/ethergo/signer/signer"
+	"github.com/synapsecns/sanguine/ethergo/submitter"
+	omnirpcClient "github.com/synapsecns/sanguine/services/omnirpc/client"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -42,6 +42,7 @@ type Notary struct {
 	summitParser                       summit.Parser
 	lastSummitBlock                    uint64
 	handler                            metrics.Handler
+	retryConfig                        []retry.WithBackoffConfigurator
 	txSubmitter                        submitter.TransactionSubmitter
 }
 
@@ -90,6 +91,13 @@ func NewNotary(ctx context.Context, cfg config.AgentConfig, omniRPCClient omnirp
 	}
 
 	notary.handler = handler
+	if cfg.MaxRetrySeconds == 0 {
+		cfg.MaxRetrySeconds = 30
+	}
+
+	notary.retryConfig = []retry.WithBackoffConfigurator{
+		retry.WithMaxAttemptTime(time.Second * time.Duration(cfg.MaxRetrySeconds)),
+	}
 
 	notary.txSubmitter = submitter.NewTransactionSubmitter(handler, notary.unbondedSigner, omniRPCClient, txDB.SubmitterDB(), &cfg.SubmitterConfig)
 
@@ -131,6 +139,20 @@ func (n *Notary) loadSummitGuardLatestStates(parentCtx context.Context) {
 
 		originID := domain.Config().DomainID
 
+		// var guardLatestState types.State
+		// contractCall := func(ctx context.Context) (err error) {
+		// 	guardLatestState, err = n.summitDomain.Summit().GetLatestState(ctx, originID)
+		// 	if err != nil {
+		// 		return fmt.Errorf("could not get latest state: %w", err)
+		// 	}
+		//
+		// 	return nil
+		// }
+		// err := retry.WithBackoff(ctx, contractCall, n.retryConfig...)
+		// if err == nil && guardLatestState.Nonce() > uint32(0) {
+		// 	n.summitGuardLatestStates[originID] = guardLatestState
+		// }
+
 		guardLatestState, err := n.summitDomain.Summit().GetLatestState(ctx, originID)
 		if err != nil {
 			guardLatestState = nil
@@ -153,12 +175,22 @@ func (n *Notary) loadNotaryLatestAttestation(parentCtx context.Context) {
 	))
 	defer span.End()
 
-	latestNotaryAttestation, err := n.summitDomain.Summit().GetLatestNotaryAttestation(ctx, n.bondedSigner)
+	var latestNotaryAttestation types.NotaryAttestation
+	contractCall := func(ctx context.Context) (err error) {
+		latestNotaryAttestation, err = n.summitDomain.Summit().GetLatestNotaryAttestation(ctx, n.bondedSigner)
+		if err != nil {
+			return fmt.Errorf("could not get latest notary attestation: %w", err)
+		}
+
+		return nil
+	}
+	err := retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
 		span.AddEvent("GetLatestNotaryAttestation failed", trace.WithAttributes(
 			attribute.String("err", err.Error()),
 		))
 	}
+
 	if latestNotaryAttestation != nil {
 		if n.myLatestNotaryAttestation == nil ||
 			latestNotaryAttestation.Attestation().SnapshotRoot() != n.myLatestNotaryAttestation.Attestation().SnapshotRoot() {
@@ -173,20 +205,39 @@ func (n *Notary) shouldNotaryRegisteredOnDestination(parentCtx context.Context)
 		attribute.Int(metrics.ChainID, int(n.destinationDomain.Config().DomainID)),
 	))
 	defer span.End()
+	var bondingManagerAgentRoot [32]byte
+	contractCall := func(ctx context.Context) (err error) {
+		bondingManagerAgentRoot, err = n.summitDomain.BondingManager().GetAgentRoot(ctx)
+		if err != nil {
+			return fmt.Errorf("could not get agent root: %w", err)
+		}
 
-	bondingManagerAgentRoot, err := n.summitDomain.BondingManager().GetAgentRoot(ctx)
+		return nil
+	}
+	err := retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
 		span.AddEvent("GetAgentRoot failed on bonding manager", trace.WithAttributes(
 			attribute.String("err", err.Error()),
 		))
+
 		return false, false
 	}
 
-	destinationLightManagerAgentRoot, err := n.destinationDomain.LightManager().GetAgentRoot(ctx)
+	var destinationLightManagerAgentRoot [32]byte
+	contractCall = func(ctx context.Context) (err error) {
+		destinationLightManagerAgentRoot, err = n.destinationDomain.LightManager().GetAgentRoot(ctx)
+		if err != nil {
+			return fmt.Errorf("could not get agent root: %w", err)
+		}
+
+		return nil
+	}
+	err = retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
 		span.AddEvent("GetAgentRoot failed on destination light manager", trace.WithAttributes(
 			attribute.String("err", err.Error()),
 		))
+
 		return false, false
 	}
 
@@ -195,13 +246,24 @@ func (n *Notary) shouldNotaryRegisteredOnDestination(parentCtx context.Context)
 		return false, false
 	}
 
-	agentStatus, err := n.destinationDomain.LightManager().GetAgentStatus(ctx, n.bondedSigner.Address())
+	var agentStatus types.AgentStatus
+	contractCall = func(ctx context.Context) (err error) {
+		agentStatus, err = n.destinationDomain.LightManager().GetAgentStatus(ctx, n.bondedSigner.Address())
+		if err != nil {
+			return fmt.Errorf("could not get agent status: %w", err)
+		}
+
+		return nil
+	}
+	err = retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
 		span.AddEvent("GetAgentStatus failed", trace.WithAttributes(
 			attribute.String("err", err.Error()),
 		))
+
 		return false, false
 	}
+
 	if agentStatus.Flag() == types.AgentFlagUnknown {
 		// Here we want to add the Notary and proceed with sending to destination
 		return true, true
@@ -227,12 +289,22 @@ func (n *Notary) checkDidSubmitNotaryLatestAttestation(parentCtx context.Context
 		return
 	}
 
-	attNonce, err := n.destinationDomain.Destination().GetAttestationNonce(ctx, n.myLatestNotaryAttestation.Attestation().SnapshotRoot())
+	var attNonce uint32
+	contractCall := func(ctx context.Context) (err error) {
+		attNonce, err = n.destinationDomain.Destination().GetAttestationNonce(ctx, n.myLatestNotaryAttestation.Attestation().SnapshotRoot())
+		if err != nil {
+			return fmt.Errorf("could not get attestation nonce: %w", err)
+		}
+
+		return nil
+	}
+	err := retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
-		span.AddEvent("checkDidSubmitNotaryLatestAttestation failed", trace.WithAttributes(
+		span.AddEvent("GetAttestationNonce failed", trace.WithAttributes(
 			attribute.String("err", err.Error()),
 		))
 	}
+
 	if attNonce > 0 {
 		n.didSubmitMyLatestNotaryAttestation = true
 	}
@@ -253,7 +325,16 @@ func (n *Notary) isValidOnOrigin(parentCtx context.Context, state types.State, d
 
 	defer span.End()
 
-	stateOnOrigin, err := domain.Origin().SuggestState(ctx, state.Nonce())
+	var stateOnOrigin types.State
+	contractCall := func(ctx context.Context) (err error) {
+		stateOnOrigin, err = domain.Origin().SuggestState(ctx, state.Nonce())
+		if err != nil {
+			return fmt.Errorf("could not suggest state: %w", err)
+		}
+
+		return nil
+	}
+	err := retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
 		span.AddEvent("SuggestState failed", trace.WithAttributes(
 			attribute.String("err", err.Error()),
@@ -412,28 +493,47 @@ func (n *Notary) submitLatestSnapshot(parentCtx context.Context) {
 	}
 }
 
-// codebeat:disable[CYCLO,DEPTH,LOC]
-//
 //nolint:cyclop
 func (n *Notary) registerNotaryOnDestination(parentCtx context.Context) bool {
 	ctx, span := n.handler.Tracer().Start(parentCtx, "registerNotaryOnDestination")
 	defer span.End()
 
-	agentProof, err := n.summitDomain.BondingManager().GetProof(ctx, n.bondedSigner.Address())
+	var agentProof [][32]byte
+	contractCall := func(ctx context.Context) (err error) {
+		agentProof, err = n.summitDomain.BondingManager().GetProof(ctx, n.bondedSigner.Address())
+		if err != nil {
+			return fmt.Errorf("could not get agent proof: %w", err)
+		}
+
+		return nil
+	}
+	err := retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
-		logger.Errorf("Error getting agent proof: %v", err)
-		span.AddEvent("Error getting agent proof", trace.WithAttributes(
+		span.AddEvent("GetProof on bonding manager failed", trace.WithAttributes(
 			attribute.String("err", err.Error()),
 		))
+
 		return false
 	}
-	agentStatus, err := n.summitDomain.BondingManager().GetAgentStatus(ctx, n.bondedSigner.Address())
+
+	var agentStatus types.AgentStatus
+	contractCall = func(ctx context.Context) (err error) {
+		agentStatus, err = n.summitDomain.BondingManager().GetAgentStatus(ctx, n.bondedSigner.Address())
+		if err != nil {
+			return fmt.Errorf("could not get agent status: %w", err)
+		}
+
+		return nil
+	}
+	err = retry.WithBackoff(ctx, contractCall, n.retryConfig...)
 	if err != nil {
 		span.AddEvent("GetAgentStatus on bonding manager failed", trace.WithAttributes(
 			attribute.String("err", err.Error()),
 		))
+
 		return false
 	}
+
 	_, err = n.txSubmitter.SubmitTransaction(ctx, big.NewInt(int64(n.destinationDomain.Config().DomainID)), func(transactor *bind.TransactOpts) (tx *ethTypes.Transaction, err error) {
 		tx, err = n.destinationDomain.LightManager().UpdateAgentStatus(
 			transactor,
@@ -500,8 +600,6 @@ func (n *Notary) submitMyLatestAttestation(parentCtx context.Context) {
 
 // Start starts the notary.
 //
-// codebeat:disable[CYCLO,DEPTH]
-//
 //nolint:cyclop
 func (n *Notary) Start(parentCtx context.Context) error {
 	g, ctx := errgroup.WithContext(parentCtx)