feat: refine some metrics

docker-compose.yml

@@ -1,6 +1,5 @@
 version: '3.8'
-volumes:
-  postgres:
+
 services:
   postgres:
     image: "postgres:16-alpine"
@@ -16,27 +15,32 @@ services:
       POSTGRES_USER: "ledger"
       POSTGRES_PASSWORD: "ledger"
       POSTGRES_DB: "ledger"
-      PGDATA: /data/postgres
-    volumes:
-    - postgres:/data/postgres
+
+  jaeger:
+    image: jaegertracing/opentelemetry-all-in-one
+    ports:
+      - "16686:16686/tcp"
 
   ledger:
     image: golang:1.22-alpine
     entrypoint: go run main.go serve
     volumes:
     - .:/src
     ports:
-    # HTTP API
     - 3068:3068
-    # pprof
-    - 3000:3000
     working_dir: /src
     depends_on:
       postgres:
         condition: service_healthy
+      jaeger:
+        condition: service_started
     environment:
-      POSTGRES_URI: "postgresql://ledger:ledger@postgres/ledger?sslmode=disable"
+      GOWORK: off
       DEBUG: "true"
       ENABLE_PPROF: "true"
-      GOWORK: off
+      OTEL_SERVICE_NAME: ledger
       OTEL_METRICS_EXPORTER: "memory"
+      OTEL_TRACES_EXPORTER: otlp
+      OTEL_TRACES_EXPORTER_OTLP_ENDPOINT: jaeger:4317
+      OTEL_TRACES_EXPORTER_OTLP_INSECURE: "true"
+      POSTGRES_URI: "postgresql://ledger:ledger@postgres/ledger?sslmode=disable"

examples/otlp-exporter/docker-compose.yml

@@ -33,11 +33,9 @@ services:
     environment:
       CGO_ENABLED: 0
       DEBUG: "true"
-      OTEL_TRACES: "true"
       OTEL_TRACES_EXPORTER: otlp
       OTEL_TRACES_EXPORTER_OTLP_ENDPOINT: otel:4317
       OTEL_TRACES_EXPORTER_OTLP_INSECURE: "true"
-      OTEL_METRICS: "true"
       OTEL_METRICS_EXPORTER: otlp
       OTEL_METRICS_EXPORTER_OTLP_ENDPOINT: otel:4317
       OTEL_METRICS_EXPORTER_OTLP_INSECURE: "true"

internal/api/v1/middleware_auto_create_ledger.go

@@ -1,6 +1,7 @@
 package v1
 
 import (
+	"go.opentelemetry.io/otel/trace"
 	"net/http"
 
 	"errors"
@@ -14,18 +15,21 @@ import (
 	"github.com/go-chi/chi/v5"
 )
 
-func autoCreateMiddleware(backend system.Controller) func(handler http.Handler) http.Handler {
+func autoCreateMiddleware(backend system.Controller, tracer trace.Tracer) func(handler http.Handler) http.Handler {
 	return func(handler http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 
+			ctx, span := tracer.Start(r.Context(), "AutomaticLedgerCreate")
+			defer span.End()
+
 			ledgerName := chi.URLParam(r, "ledger")
-			if _, err := backend.GetLedger(r.Context(), ledgerName); err != nil {
+			if _, err := backend.GetLedger(ctx, ledgerName); err != nil {
 				if !postgres.IsNotFoundError(err) {
 					api.InternalServerError(w, r, err)
 					return
 				}
 
-				if err := backend.CreateLedger(r.Context(), ledgerName, ledger.Configuration{
+				if err := backend.CreateLedger(ctx, ledgerName, ledger.Configuration{
 					Bucket: ledgerName,
 				}); err != nil {
 					switch {
@@ -37,6 +41,7 @@ func autoCreateMiddleware(backend system.Controller) func(handler http.Handler)
 					return
 				}
 			}
+			span.End()
 
 			handler.ServeHTTP(w, r)
 		})

internal/api/v1/routes.go

@@ -45,7 +45,7 @@ func NewRouter(
 					handler.ServeHTTP(w, r)
 				})
 			})
-			router.Use(autoCreateMiddleware(systemController))
+			router.Use(autoCreateMiddleware(systemController, routerOptions.tracer))
 			router.Use(common.LedgerMiddleware(systemController, func(r *http.Request) string {
 				return chi.URLParam(r, "ledger")
 			}, routerOptions.tracer, "/_info"))
@@ -98,4 +98,4 @@ func WithTracer(tracer trace.Tracer) RouterOption {
 
 var defaultRouterOptions = []RouterOption{
 	WithTracer(nooptracer.Tracer{}),
-}
+}

internal/controller/ledger/controller_with_cache.go

@@ -2,6 +2,8 @@ package ledger
 
 import (
 	"context"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	ledger "github.com/formancehq/ledger/internal"
 )
@@ -15,6 +17,7 @@ type ControllerWithCache struct {
 func (c *ControllerWithCache) IsDatabaseUpToDate(ctx context.Context) (bool, error) {
 
 	if c.registry.IsUpToDate(c.ledger.Name) {
+		trace.SpanFromContext(ctx).SetAttributes(attribute.Bool("cache-hit", true))
 		return true, nil
 	}
 

internal/controller/ledger/controller_with_too_many_client_handling.go

@@ -84,7 +84,7 @@ func handleRetry[INPUT, OUTPUT any](
 	fn func(ctx context.Context, parameters Parameters[INPUT]) (*OUTPUT, error),
 ) (*OUTPUT, error) {
 
-	ctx, span := tracer.Start(ctx, "Retrier")
+	ctx, span := tracer.Start(ctx, "TooManyClientRetrier")
 	defer span.End()
 
 	count := 0

internal/storage/ledger/accounts.go

@@ -331,6 +331,8 @@ func (s *Store) DeleteAccountMetadata(ctx context.Context, account, key string)
 	return err
 }
 
+// todo: since we update first balances of an accounts in the transaction process, we can avoid nested sql txs
+// while upserting account and upsert them all in one shot
 func (s *Store) UpsertAccount(ctx context.Context, account *ledger.Account) (bool, error) {
 	return tracing.TraceWithMetric(
 		ctx,

internal/storage/ledger/logs.go

@@ -47,21 +47,21 @@ func (j RawMessage) Value() (driver.Value, error) {
 
 func (s *Store) InsertLog(ctx context.Context, log *ledger.Log) error {
 
-	// We lock logs table as we need than the last log does not change until the transaction commit
-	if s.ledger.HasFeature(ledger.FeatureHashLogs, "SYNC") {
-		_, err := s.db.NewRaw(`select pg_advisory_xact_lock(?)`, s.ledger.ID).Exec(ctx)
-		if err != nil {
-			return postgres.ResolveError(err)
-		}
-	}
-
 	_, err := tracing.TraceWithMetric(
 		ctx,
 		"InsertLog",
 		s.tracer,
 		s.insertLogHistogram,
 		tracing.NoResult(func(ctx context.Context) error {
 
+			// We lock logs table as we need than the last log does not change until the transaction commit
+			if s.ledger.HasFeature(ledger.FeatureHashLogs, "SYNC") {
+				_, err := s.db.NewRaw(`select pg_advisory_xact_lock(?)`, s.ledger.ID).Exec(ctx)
+				if err != nil {
+					return postgres.ResolveError(err)
+				}
+			}
+
 			payloadData, err := json.Marshal(log.Data)
 			if err != nil {
 				return fmt.Errorf("failed to marshal log data: %w", err)

test/performance/benchmark_test.go

@@ -87,7 +87,7 @@ func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Result {
 				report.End = time.Now()
 
 				// Fetch otel metrics
-				rsp, err := http.Get(env.URL() + "/_metrics")
+				rsp, err := http.Get(env.URL() + "/_/metrics")
 				require.NoError(b, err)
 				ret := make(map[string]any)
 				require.NoError(b, json.NewDecoder(rsp.Body).Decode(&ret))