From 20bda717821f25c7f091bf49215e95cb729f92bb Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 16:17:10 -0400 Subject: [PATCH 01/12] [goreleaser] observability hotfix --- services/rfq/relayer/service/handlers.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/services/rfq/relayer/service/handlers.go b/services/rfq/relayer/service/handlers.go index e67ed15de4..2cfa1b600e 100644 --- a/services/rfq/relayer/service/handlers.go +++ b/services/rfq/relayer/service/handlers.go @@ -20,7 +20,7 @@ import ( "go.opentelemetry.io/otel/trace" ) -var maxRPCRetryTime = 15 * time.Second +var maxRPCRetryTime = 30 * time.Second // handleBridgeRequestedLog handles the BridgeRequestedLog event. // Step 1: Seen @@ -71,7 +71,16 @@ func (r *Relayer) handleBridgeRequestedLog(parentCtx context.Context, req *fastb } var bridgeTx fastbridge.IFastBridgeBridgeTransaction - call := func(ctx context.Context) error { + call := func(parentCtx context.Context) error { + ctx, newSpan := r.metrics.Tracer().Start(parentCtx, "fetchBridgeTx", trace.WithAttributes( + attribute.String("transaction_id", hexutil.Encode(req.TransactionId[:])), + attribute.String("bridge_address", req.Raw.Address.String()), + )) + + defer func() { + metrics.EndSpanWithErr(newSpan, err) + }() + bridgeTx, err = fastBridge.GetBridgeTransaction(&bind.CallOpts{Context: ctx}, req.Request) if err != nil { return fmt.Errorf("could not get bridge transaction: %w", err) From 160a5b2c78ee10c66e2f620552d1e42c4fca4582 Mon Sep 17 00:00:00 2001 From: Daniel Wasserman Date: Wed, 17 Jul 2024 15:34:13 -0500 Subject: [PATCH 02/12] Feat: record err as attribute --- core/metrics/spanutils.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/metrics/spanutils.go b/core/metrics/spanutils.go index 865c4f6541..d370063846 100644 --- a/core/metrics/spanutils.go +++ b/core/metrics/spanutils.go @@ -1,6 +1,9 @@ package metrics -import "go.opentelemetry.io/otel/trace" +import ( + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) // EndSpanWithErr ends a span and records an error if one is present. func EndSpanWithErr(span trace.Span, err error) { @@ -9,6 +12,7 @@ func EndSpanWithErr(span trace.Span, err error) { } if err != nil { + span.SetAttributes(attribute.String("span_error", err.Error())) span.RecordError(err) } From dff84eb209298767a52f299684298c2f54830d79 Mon Sep 17 00:00:00 2001 From: Daniel Wasserman Date: Wed, 17 Jul 2024 15:34:20 -0500 Subject: [PATCH 03/12] [goreleaser] From 690b1670e8cae7c27ae7b2340d30534c31bfca42 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 16:36:31 -0400 Subject: [PATCH 04/12] [goreleaser] --- services/rfq/guard/service/handlers.go | 9 +++++++++ services/rfq/relayer/service/handlers.go | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/services/rfq/guard/service/handlers.go b/services/rfq/guard/service/handlers.go index 736fd61b5a..e7d33123bf 100644 --- a/services/rfq/guard/service/handlers.go +++ b/services/rfq/guard/service/handlers.go @@ -43,6 +43,15 @@ func (g *Guard) handleBridgeRequestedLog(parentCtx context.Context, req *fastbri var bridgeTx fastbridge.IFastBridgeBridgeTransaction call := func(ctx context.Context) error { + ctx, newSpan := g.metrics.Tracer().Start(parentCtx, "fetchBridgeTx", trace.WithAttributes( + attribute.String("transaction_id", hexutil.Encode(req.TransactionId[:])), + attribute.String("bridge_address", req.Raw.Address.String()), + )) + + defer func() { + metrics.EndSpanWithErr(newSpan, err) + }() + bridgeTx, err = fastBridge.GetBridgeTransaction(&bind.CallOpts{Context: ctx}, req.Request) if err != nil { return fmt.Errorf("could not get bridge transaction: %w", err) diff --git a/services/rfq/relayer/service/handlers.go b/services/rfq/relayer/service/handlers.go index 2cfa1b600e..11017b9342 100644 --- a/services/rfq/relayer/service/handlers.go +++ b/services/rfq/relayer/service/handlers.go @@ -430,6 +430,14 @@ func (q *QuoteRequestHandler) handleProofPosted(ctx context.Context, _ trace.Spa var canClaim bool claimCall := func(ctx context.Context) error { + ctx, newSpan := q.metrics.Tracer().Start(ctx, "checkClaimCheck", trace.WithAttributes( + attribute.String("transaction_id", hexutil.Encode(request.TransactionID[:])), + )) + + defer func() { + metrics.EndSpanWithErr(newSpan, err) + }() + canClaim, err = q.Origin.Bridge.CanClaim(&bind.CallOpts{Context: ctx}, request.TransactionID, q.RelayerAddress) if err != nil { return fmt.Errorf("could not check if can claim: %w", err) From 2358e16480720fb05fb548524e2560efdac39537 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 17:04:04 -0400 Subject: [PATCH 05/12] order hotfix [goreleaser] --- services/rfq/relayer/service/handlers.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/services/rfq/relayer/service/handlers.go b/services/rfq/relayer/service/handlers.go index 11017b9342..b16070e6a1 100644 --- a/services/rfq/relayer/service/handlers.go +++ b/services/rfq/relayer/service/handlers.go @@ -341,14 +341,15 @@ func (r *Relayer) handleRelayLog(ctx context.Context, req *fastbridge.FastBridge } // TODO: this can still get re-orged - err = r.db.UpdateQuoteRequestStatus(ctx, req.TransactionId, reldb.RelayCompleted, nil) - if err != nil { - return fmt.Errorf("could not update request status: %w", err) - } err = r.db.UpdateDestTxHash(ctx, req.TransactionId, req.Raw.TxHash) if err != nil { return fmt.Errorf("could not update dest tx hash: %w", err) } + + err = r.db.UpdateQuoteRequestStatus(ctx, req.TransactionId, reldb.RelayCompleted, nil) + if err != nil { + return fmt.Errorf("could not update request status: %w", err) + } return nil } From 72dbf2bc6125eb2c2745fc5114602d4fbf50bb33 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 17:19:18 -0400 Subject: [PATCH 06/12] [goreleaser] error accumulator --- core/retry/retry.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/retry/retry.go b/core/retry/retry.go index 2f25007e39..e96a077c17 100644 --- a/core/retry/retry.go +++ b/core/retry/retry.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" "github.com/jpillora/backoff" - errorUtil "github.com/pkg/errors" + "go.uber.org/multierr" "time" ) @@ -118,6 +118,8 @@ func WithBackoff(ctx context.Context, doFunc RetryableFunc, configurators ...Wit timeout := time.Duration(0) startTime := time.Now() + var errs []error + attempts := 0 for !config.exceedsMaxAttempts(attempts) && !config.exceedsMaxTime(startTime) { select { @@ -135,6 +137,7 @@ func WithBackoff(ctx context.Context, doFunc RetryableFunc, configurators ...Wit err := doFunc(funcCtx) if err != nil { + errs = append(errs, err) timeout = b.Duration() attempts++ cancel() @@ -146,10 +149,10 @@ func WithBackoff(ctx context.Context, doFunc RetryableFunc, configurators ...Wit } if config.exceedsMaxAttempts(attempts) { - return errorUtil.Wrapf(ErrMaxAttempts, "after %d attempts", attempts) + return multierr.Append(ErrMaxAttempts, fmt.Errorf("after %d attempts (attempt errors: %w)", attempts, multierr.Combine(errs...))) } if config.exceedsMaxTime(startTime) { - return errorUtil.Wrapf(ErrMaxTime, "after %s (max was %s)", time.Since(startTime).String(), config.maxAllAttemptsTime.String()) + return multierr.Append(ErrMaxTime, fmt.Errorf("after %s (max was %s) %w", time.Since(startTime).String(), config.maxAllAttemptsTime.String(), multierr.Combine(errs...))) } return ErrUnknown From 9f409edc888e2c2e8d30858cb19f9e6d5e477a21 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 17:25:36 -0400 Subject: [PATCH 07/12] [goreleaser] --- core/go.mod | 2 +- core/retry/retry.go | 2 ++ services/rfq/guard/service/handlers.go | 9 --------- services/rfq/relayer/service/handlers.go | 9 --------- 4 files changed, 3 insertions(+), 19 deletions(-) diff --git a/core/go.mod b/core/go.mod index 6c32da15ba..9d57309b76 100644 --- a/core/go.mod +++ b/core/go.mod @@ -56,6 +56,7 @@ require ( go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/sdk/metric v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 + go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/sync v0.7.0 gorm.io/driver/sqlite v1.5.6 @@ -163,7 +164,6 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opentelemetry.io/otel/log v0.3.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect - go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.24.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect diff --git a/core/retry/retry.go b/core/retry/retry.go index e96a077c17..ee93c847df 100644 --- a/core/retry/retry.go +++ b/core/retry/retry.go @@ -149,9 +149,11 @@ func WithBackoff(ctx context.Context, doFunc RetryableFunc, configurators ...Wit } if config.exceedsMaxAttempts(attempts) { + // nolint: wrapcheck return multierr.Append(ErrMaxAttempts, fmt.Errorf("after %d attempts (attempt errors: %w)", attempts, multierr.Combine(errs...))) } if config.exceedsMaxTime(startTime) { + // nolint: wrapcheck return multierr.Append(ErrMaxTime, fmt.Errorf("after %s (max was %s) %w", time.Since(startTime).String(), config.maxAllAttemptsTime.String(), multierr.Combine(errs...))) } diff --git a/services/rfq/guard/service/handlers.go b/services/rfq/guard/service/handlers.go index e7d33123bf..736fd61b5a 100644 --- a/services/rfq/guard/service/handlers.go +++ b/services/rfq/guard/service/handlers.go @@ -43,15 +43,6 @@ func (g *Guard) handleBridgeRequestedLog(parentCtx context.Context, req *fastbri var bridgeTx fastbridge.IFastBridgeBridgeTransaction call := func(ctx context.Context) error { - ctx, newSpan := g.metrics.Tracer().Start(parentCtx, "fetchBridgeTx", trace.WithAttributes( - attribute.String("transaction_id", hexutil.Encode(req.TransactionId[:])), - attribute.String("bridge_address", req.Raw.Address.String()), - )) - - defer func() { - metrics.EndSpanWithErr(newSpan, err) - }() - bridgeTx, err = fastBridge.GetBridgeTransaction(&bind.CallOpts{Context: ctx}, req.Request) if err != nil { return fmt.Errorf("could not get bridge transaction: %w", err) diff --git a/services/rfq/relayer/service/handlers.go b/services/rfq/relayer/service/handlers.go index b16070e6a1..1f1cc52b6f 100644 --- a/services/rfq/relayer/service/handlers.go +++ b/services/rfq/relayer/service/handlers.go @@ -72,15 +72,6 @@ func (r *Relayer) handleBridgeRequestedLog(parentCtx context.Context, req *fastb var bridgeTx fastbridge.IFastBridgeBridgeTransaction call := func(parentCtx context.Context) error { - ctx, newSpan := r.metrics.Tracer().Start(parentCtx, "fetchBridgeTx", trace.WithAttributes( - attribute.String("transaction_id", hexutil.Encode(req.TransactionId[:])), - attribute.String("bridge_address", req.Raw.Address.String()), - )) - - defer func() { - metrics.EndSpanWithErr(newSpan, err) - }() - bridgeTx, err = fastBridge.GetBridgeTransaction(&bind.CallOpts{Context: ctx}, req.Request) if err != nil { return fmt.Errorf("could not get bridge transaction: %w", err) From 4951dc79baed6cc865b4ac511b375195c82d5ba0 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 17:28:53 -0400 Subject: [PATCH 08/12] [goreleaser] --- services/rfq/relayer/service/handlers.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/services/rfq/relayer/service/handlers.go b/services/rfq/relayer/service/handlers.go index 1f1cc52b6f..05bb14681c 100644 --- a/services/rfq/relayer/service/handlers.go +++ b/services/rfq/relayer/service/handlers.go @@ -422,14 +422,6 @@ func (q *QuoteRequestHandler) handleProofPosted(ctx context.Context, _ trace.Spa var canClaim bool claimCall := func(ctx context.Context) error { - ctx, newSpan := q.metrics.Tracer().Start(ctx, "checkClaimCheck", trace.WithAttributes( - attribute.String("transaction_id", hexutil.Encode(request.TransactionID[:])), - )) - - defer func() { - metrics.EndSpanWithErr(newSpan, err) - }() - canClaim, err = q.Origin.Bridge.CanClaim(&bind.CallOpts{Context: ctx}, request.TransactionID, q.RelayerAddress) if err != nil { return fmt.Errorf("could not check if can claim: %w", err) From 312e9353bc68214272ee1256511d50cb4da358a6 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 17:32:18 -0400 Subject: [PATCH 09/12] [goreleaser] again --- services/rfq/relayer/service/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/rfq/relayer/service/handlers.go b/services/rfq/relayer/service/handlers.go index 05bb14681c..d606bce5b9 100644 --- a/services/rfq/relayer/service/handlers.go +++ b/services/rfq/relayer/service/handlers.go @@ -71,7 +71,7 @@ func (r *Relayer) handleBridgeRequestedLog(parentCtx context.Context, req *fastb } var bridgeTx fastbridge.IFastBridgeBridgeTransaction - call := func(parentCtx context.Context) error { + call := func(ctx context.Context) error { bridgeTx, err = fastBridge.GetBridgeTransaction(&bind.CallOpts{Context: ctx}, req.Request) if err != nil { return fmt.Errorf("could not get bridge transaction: %w", err) From 53067c3e02321842d19b91357b174c74871b47ca Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 17:47:56 -0400 Subject: [PATCH 10/12] test out namespace --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index de5090c565..0be669c9c3 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -77,7 +77,7 @@ jobs: test: name: Go Coverage - runs-on: ${{ matrix.platform }} + runs-on: namespace-profile-default #${{ matrix.platform }} if: ${{ needs.changes.outputs.package_count_deps > 0 && (github.event_name != 'push' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) }} needs: changes strategy: From 5504b3269c2db91b59a7366cd8f3e96dc15a46a6 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 17:56:53 -0400 Subject: [PATCH 11/12] [goreleaser] --- .github/workflows/goreleaser-actions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/goreleaser-actions.yml b/.github/workflows/goreleaser-actions.yml index 60af3fd5d8..6fc5817246 100644 --- a/.github/workflows/goreleaser-actions.yml +++ b/.github/workflows/goreleaser-actions.yml @@ -111,7 +111,7 @@ jobs: # TODO: we may want to dry run this on prs run-goreleaser: - runs-on: ubuntu-latest + runs-on: namespace-profile-default needs: [build-goreleaser,changes] if: ${{ needs.changes.outputs.package_count > 0 }} permissions: From 35baeea202c17ad613837135ea4a694d84e9a5b5 Mon Sep 17 00:00:00 2001 From: Trajan0x Date: Wed, 17 Jul 2024 18:08:55 -0400 Subject: [PATCH 12/12] fix --- .github/workflows/go.yml | 2 +- .github/workflows/goreleaser-actions.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 0be669c9c3..de5090c565 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -77,7 +77,7 @@ jobs: test: name: Go Coverage - runs-on: namespace-profile-default #${{ matrix.platform }} + runs-on: ${{ matrix.platform }} if: ${{ needs.changes.outputs.package_count_deps > 0 && (github.event_name != 'push' || github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) }} needs: changes strategy: diff --git a/.github/workflows/goreleaser-actions.yml b/.github/workflows/goreleaser-actions.yml index 6fc5817246..60af3fd5d8 100644 --- a/.github/workflows/goreleaser-actions.yml +++ b/.github/workflows/goreleaser-actions.yml @@ -111,7 +111,7 @@ jobs: # TODO: we may want to dry run this on prs run-goreleaser: - runs-on: namespace-profile-default + runs-on: ubuntu-latest needs: [build-goreleaser,changes] if: ${{ needs.changes.outputs.package_count > 0 }} permissions: