update dest tx hash before status/improve obversability (#2888)

* [goreleaser] observability hotfix * Feat: record err as attribute * [goreleaser] * [goreleaser] * order hotfix [goreleaser] * [goreleaser] error accumulator * [goreleaser] * [goreleaser] * [goreleaser] again * test out namespace * [goreleaser] * fix --------- Co-authored-by: Trajan0x <[email protected]> Co-authored-by: Daniel Wasserman <[email protected]>
synapsecns · Jul 19, 2024 · c92fd80 · c92fd80
1 parent 788893f
commit c92fd80
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 10 deletions.
diff --git a/core/go.mod b/core/go.mod
@@ -56,6 +56,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.28.0
 	go.opentelemetry.io/otel/sdk/metric v1.28.0
 	go.opentelemetry.io/otel/trace v1.28.0
+	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/sync v0.7.0
 	gorm.io/driver/sqlite v1.5.6
@@ -163,7 +164,6 @@ require (
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opentelemetry.io/otel/log v0.3.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
-	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/arch v0.8.0 // indirect
 	golang.org/x/crypto v0.24.0 // indirect
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect

diff --git a/core/metrics/spanutils.go b/core/metrics/spanutils.go
@@ -1,6 +1,9 @@
 package metrics
 
-import "go.opentelemetry.io/otel/trace"
+import (
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
+)
 
 // EndSpanWithErr ends a span and records an error if one is present.
 func EndSpanWithErr(span trace.Span, err error) {
@@ -9,6 +12,7 @@ func EndSpanWithErr(span trace.Span, err error) {
 	}
 
 	if err != nil {
+		span.SetAttributes(attribute.String("span_error", err.Error()))
 		span.RecordError(err)
 	}
 

diff --git a/core/retry/retry.go b/core/retry/retry.go
@@ -5,7 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/jpillora/backoff"
-	errorUtil "github.com/pkg/errors"
+	"go.uber.org/multierr"
 	"time"
 )
 
@@ -118,6 +118,8 @@ func WithBackoff(ctx context.Context, doFunc RetryableFunc, configurators ...Wit
 	timeout := time.Duration(0)
 	startTime := time.Now()
 
+	var errs []error
+
 	attempts := 0
 	for !config.exceedsMaxAttempts(attempts) && !config.exceedsMaxTime(startTime) {
 		select {
@@ -135,6 +137,7 @@ func WithBackoff(ctx context.Context, doFunc RetryableFunc, configurators ...Wit
 
 			err := doFunc(funcCtx)
 			if err != nil {
+				errs = append(errs, err)
 				timeout = b.Duration()
 				attempts++
 				cancel()
@@ -146,10 +149,12 @@ func WithBackoff(ctx context.Context, doFunc RetryableFunc, configurators ...Wit
 	}
 
 	if config.exceedsMaxAttempts(attempts) {
-		return errorUtil.Wrapf(ErrMaxAttempts, "after %d attempts", attempts)
+		// nolint: wrapcheck
+		return multierr.Append(ErrMaxAttempts, fmt.Errorf("after %d attempts (attempt errors: %w)", attempts, multierr.Combine(errs...)))
 	}
 	if config.exceedsMaxTime(startTime) {
-		return errorUtil.Wrapf(ErrMaxTime, "after %s (max was %s)", time.Since(startTime).String(), config.maxAllAttemptsTime.String())
+		// nolint: wrapcheck
+		return multierr.Append(ErrMaxTime, fmt.Errorf("after %s (max was %s) %w", time.Since(startTime).String(), config.maxAllAttemptsTime.String(), multierr.Combine(errs...)))
 	}
 
 	return ErrUnknown

diff --git a/services/rfq/relayer/service/handlers.go b/services/rfq/relayer/service/handlers.go
@@ -20,7 +20,7 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
-var maxRPCRetryTime = 15 * time.Second
+var maxRPCRetryTime = 30 * time.Second
 
 // handleBridgeRequestedLog handles the BridgeRequestedLog event.
 // Step 1: Seen
@@ -332,14 +332,15 @@ func (r *Relayer) handleRelayLog(ctx context.Context, req *fastbridge.FastBridge
 	}
 
 	// TODO: this can still get re-orged
-	err = r.db.UpdateQuoteRequestStatus(ctx, req.TransactionId, reldb.RelayCompleted, nil)
-	if err != nil {
-		return fmt.Errorf("could not update request status: %w", err)
-	}
 	err = r.db.UpdateDestTxHash(ctx, req.TransactionId, req.Raw.TxHash)
 	if err != nil {
 		return fmt.Errorf("could not update dest tx hash: %w", err)
 	}
+
+	err = r.db.UpdateQuoteRequestStatus(ctx, req.TransactionId, reldb.RelayCompleted, nil)
+	if err != nil {
+		return fmt.Errorf("could not update request status: %w", err)
+	}
 	return nil
 }