RFQ Guard: add prometheus metrics for disputes (#2894)

* Feat: add otel recorder for guard

* Feat: call RecordDispute() before submitting tx

* [goreleaser]

* Fix: build

* [goreleaser]

* Feat: load disputes from db instead of cache

* [goreleaser]

* Fix: build

* [goreleaser]

* WIP: dispute logs

* [goreleaser]

* Revert "WIP: dispute logs"

This reverts commit 9607b99.

* Feat: add mux for metrics db query

---------

Co-authored-by: Trajan0x <[email protected]>

services/rfq/guard/service/guard.go

@@ -36,6 +36,7 @@ type Guard struct {
 	chainListeners map[int]listener.ContractListener
 	contracts      map[int]*fastbridge.FastBridgeRef
 	txSubmitter    submitter.TransactionSubmitter
+	otelRecorder   iOtelRecorder
 }
 
 // NewGuard creates a new Guard.
@@ -96,6 +97,11 @@ func NewGuard(ctx context.Context, metricHandler metrics.Handler, cfg guardconfi
 		txSubmitter = submitter.NewTransactionSubmitter(metricHandler, sg, omniClient, store.SubmitterDB(), &cfg.SubmitterConfig)
 	}
 
+	otelRecorder, err := newOtelRecorder(metricHandler, txSubmitter.Address(), store)
+	if err != nil {
+		return nil, fmt.Errorf("could not get otel recorder: %w", err)
+	}
+
 	return &Guard{
 		cfg:            cfg,
 		metrics:        metricHandler,
@@ -104,6 +110,7 @@ func NewGuard(ctx context.Context, metricHandler metrics.Handler, cfg guardconfi
 		chainListeners: chainListeners,
 		contracts:      contracts,
 		txSubmitter:    txSubmitter,
+		otelRecorder:   otelRecorder,
 	}, nil
 }
 

services/rfq/guard/service/otel.go

@@ -0,0 +1,85 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/synapsecns/sanguine/core/metrics"
+	"github.com/synapsecns/sanguine/services/rfq/guard/guarddb"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+const meterName = "github.com/synapsecns/sanguine/services/rfq/guard/service"
+
+// generate an interface for otelRecorder that exports the public method.
+// this allows us to avoid using recordX externally anad makes the package less confusing.
+//
+// =============================================================================
+// =============================================================================
+// IMPORTANT: DO NOT REMOVE THIS COMMENT.
+// NOTICE: PLEASE MAKE SURE YOU UPDATE BOTH THE DOCS AND THE GRAFANA DASHBOARD (IF NEEDED) AFTER UPDATING METRICS.
+// =============================================================================
+// =============================================================================
+//
+//go:generate go run github.com/vburenin/ifacemaker -f otel.go -s otelRecorder -i iOtelRecorder -p service -o otel_generated.go -c "autogenerated file"
+type otelRecorder struct {
+	metrics metrics.Handler
+	// meter is the metrics meter.
+	meter metric.Meter
+	// disputeGauge is the gauge for the disputeCounts.
+	disputeGauge metric.Int64ObservableGauge
+	// db is a database connection for fetching disputes.
+	db guarddb.Service
+	// walletAddress is the wallet address for the signer.
+	walletAddress common.Address
+	// mux is a mutex used for making sure only one db query is run at a time.
+	mux sync.Mutex
+}
+
+func newOtelRecorder(meterHandler metrics.Handler, walletAddress common.Address, db guarddb.Service) (_ iOtelRecorder, err error) {
+	or := otelRecorder{
+		metrics:       meterHandler,
+		meter:         meterHandler.Meter(meterName),
+		db:            db,
+		walletAddress: walletAddress,
+	}
+
+	or.disputeGauge, err = or.meter.Int64ObservableGauge("dispute_count")
+	if err != nil {
+		return nil, fmt.Errorf("could not create dispute count gauge: %w", err)
+	}
+
+	_, err = or.meter.RegisterCallback(or.recordDisputeCounts, or.disputeGauge)
+	if err != nil {
+		return nil, fmt.Errorf("could not register callback for dispute count gauge: %w", err)
+	}
+
+	return &or, nil
+}
+
+func (o *otelRecorder) recordDisputeCounts(ctx context.Context, observer metric.Observer) (err error) {
+	if o.metrics == nil || o.disputeGauge == nil || o.db == nil {
+		return nil
+	}
+
+	// make sure we aren't making simultaneous db queries
+	if o.mux.TryLock() {
+		defer o.mux.Unlock()
+	} else {
+		return nil
+	}
+
+	results, err := o.db.GetPendingProvensByStatus(ctx, guarddb.DisputePending, guarddb.Disputed)
+	if err != nil {
+		return fmt.Errorf("could not get disputes: %w", err)
+	}
+	opts := metric.WithAttributes(
+		attribute.String("wallet", o.walletAddress.Hex()),
+	)
+	observer.ObserveInt64(o.disputeGauge, int64(len(results)), opts)
+
+	return nil
+}