From 3e9e94fb1d5a4d5a6f98be4974b45635c916c5c7 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Sat, 12 Oct 2024 22:46:48 +0200 Subject: [PATCH] feat: refine some metrics --- docker-compose.yml | 24 ++++++++------ examples/jaeger-exporter/docker-compose.yml | 33 ------------------- examples/otlp-exporter/docker-compose.yml | 2 -- .../api/v1/middleware_auto_create_ledger.go | 11 +++++-- internal/api/v1/routes.go | 4 +-- .../ledger/controller_with_cache.go | 3 ++ ...ontroller_with_too_many_client_handling.go | 2 +- internal/storage/ledger/accounts.go | 2 ++ internal/storage/ledger/logs.go | 16 ++++----- test/performance/benchmark_test.go | 2 +- 10 files changed, 39 insertions(+), 60 deletions(-) delete mode 100644 examples/jaeger-exporter/docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml index 40b88bf0b..0f328ac55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,5 @@ version: '3.8' -volumes: - postgres: + services: postgres: image: "postgres:16-alpine" @@ -16,9 +15,11 @@ services: POSTGRES_USER: "ledger" POSTGRES_PASSWORD: "ledger" POSTGRES_DB: "ledger" - PGDATA: /data/postgres - volumes: - - postgres:/data/postgres + + jaeger: + image: jaegertracing/opentelemetry-all-in-one + ports: + - "16686:16686/tcp" ledger: image: golang:1.22-alpine @@ -26,17 +27,20 @@ services: volumes: - .:/src ports: - # HTTP API - 3068:3068 - # pprof - - 3000:3000 working_dir: /src depends_on: postgres: condition: service_healthy + jaeger: + condition: service_started environment: - POSTGRES_URI: "postgresql://ledger:ledger@postgres/ledger?sslmode=disable" + GOWORK: off DEBUG: "true" ENABLE_PPROF: "true" - GOWORK: off + OTEL_SERVICE_NAME: ledger OTEL_METRICS_EXPORTER: "memory" + OTEL_TRACES_EXPORTER: otlp + OTEL_TRACES_EXPORTER_OTLP_ENDPOINT: jaeger:4317 + OTEL_TRACES_EXPORTER_OTLP_INSECURE: "true" + POSTGRES_URI: "postgresql://ledger:ledger@postgres/ledger?sslmode=disable" diff --git a/examples/jaeger-exporter/docker-compose.yml b/examples/jaeger-exporter/docker-compose.yml deleted file mode 100644 index 746f1db65..000000000 --- a/examples/jaeger-exporter/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -volumes: - postgres: -services: - postgres: - extends: - file: ../../docker-compose.yml - service: postgres - jaeger: - image: jaegertracing/opentelemetry-all-in-one - ports: - - "16686:16686/tcp" - ledger: - extends: - file: ../../docker-compose.yml - service: ledger - depends_on: - - postgres - - jaeger - image: golang:1.19-alpine - entrypoint: go run main.go serve - volumes: - - ../..:/src - working_dir: /src - ports: - - "3068:3068/tcp" - environment: - CGO_ENABLED: 0 - DEBUG: "true" - OTEL_TRACES: "true" - OTEL_TRACES_EXPORTER: jaeger - OTEL_TRACES_EXPORTER_JAEGER_ENDPOINT: http://jaeger:14268/api/traces - OTEL_SERVICE_NAME: ledger diff --git a/examples/otlp-exporter/docker-compose.yml b/examples/otlp-exporter/docker-compose.yml index 3a6ea3b5b..753a295ee 100644 --- a/examples/otlp-exporter/docker-compose.yml +++ b/examples/otlp-exporter/docker-compose.yml @@ -33,11 +33,9 @@ services: environment: CGO_ENABLED: 0 DEBUG: "true" - OTEL_TRACES: "true" OTEL_TRACES_EXPORTER: otlp OTEL_TRACES_EXPORTER_OTLP_ENDPOINT: otel:4317 OTEL_TRACES_EXPORTER_OTLP_INSECURE: "true" - OTEL_METRICS: "true" OTEL_METRICS_EXPORTER: otlp OTEL_METRICS_EXPORTER_OTLP_ENDPOINT: otel:4317 OTEL_METRICS_EXPORTER_OTLP_INSECURE: "true" diff --git a/internal/api/v1/middleware_auto_create_ledger.go b/internal/api/v1/middleware_auto_create_ledger.go index f3240a333..62c5b597a 100644 --- a/internal/api/v1/middleware_auto_create_ledger.go +++ b/internal/api/v1/middleware_auto_create_ledger.go @@ -1,6 +1,7 @@ package v1 import ( + "go.opentelemetry.io/otel/trace" "net/http" "errors" @@ -14,18 +15,21 @@ import ( "github.com/go-chi/chi/v5" ) -func autoCreateMiddleware(backend system.Controller) func(handler http.Handler) http.Handler { +func autoCreateMiddleware(backend system.Controller, tracer trace.Tracer) func(handler http.Handler) http.Handler { return func(handler http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, span := tracer.Start(r.Context(), "AutomaticLedgerCreate") + defer span.End() + ledgerName := chi.URLParam(r, "ledger") - if _, err := backend.GetLedger(r.Context(), ledgerName); err != nil { + if _, err := backend.GetLedger(ctx, ledgerName); err != nil { if !postgres.IsNotFoundError(err) { api.InternalServerError(w, r, err) return } - if err := backend.CreateLedger(r.Context(), ledgerName, ledger.Configuration{ + if err := backend.CreateLedger(ctx, ledgerName, ledger.Configuration{ Bucket: ledgerName, }); err != nil { switch { @@ -37,6 +41,7 @@ func autoCreateMiddleware(backend system.Controller) func(handler http.Handler) return } } + span.End() handler.ServeHTTP(w, r) }) diff --git a/internal/api/v1/routes.go b/internal/api/v1/routes.go index d9499281a..a27419918 100644 --- a/internal/api/v1/routes.go +++ b/internal/api/v1/routes.go @@ -45,7 +45,7 @@ func NewRouter( handler.ServeHTTP(w, r) }) }) - router.Use(autoCreateMiddleware(systemController)) + router.Use(autoCreateMiddleware(systemController, routerOptions.tracer)) router.Use(common.LedgerMiddleware(systemController, func(r *http.Request) string { return chi.URLParam(r, "ledger") }, routerOptions.tracer, "/_info")) @@ -98,4 +98,4 @@ func WithTracer(tracer trace.Tracer) RouterOption { var defaultRouterOptions = []RouterOption{ WithTracer(nooptracer.Tracer{}), -} \ No newline at end of file +} diff --git a/internal/controller/ledger/controller_with_cache.go b/internal/controller/ledger/controller_with_cache.go index 1e68ec8ba..ec5f7532b 100644 --- a/internal/controller/ledger/controller_with_cache.go +++ b/internal/controller/ledger/controller_with_cache.go @@ -2,6 +2,8 @@ package ledger import ( "context" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ledger "github.com/formancehq/ledger/internal" ) @@ -15,6 +17,7 @@ type ControllerWithCache struct { func (c *ControllerWithCache) IsDatabaseUpToDate(ctx context.Context) (bool, error) { if c.registry.IsUpToDate(c.ledger.Name) { + trace.SpanFromContext(ctx).SetAttributes(attribute.Bool("cache-hit", true)) return true, nil } diff --git a/internal/controller/ledger/controller_with_too_many_client_handling.go b/internal/controller/ledger/controller_with_too_many_client_handling.go index 17f3ca52c..9f7e20b19 100644 --- a/internal/controller/ledger/controller_with_too_many_client_handling.go +++ b/internal/controller/ledger/controller_with_too_many_client_handling.go @@ -84,7 +84,7 @@ func handleRetry[INPUT, OUTPUT any]( fn func(ctx context.Context, parameters Parameters[INPUT]) (*OUTPUT, error), ) (*OUTPUT, error) { - ctx, span := tracer.Start(ctx, "Retrier") + ctx, span := tracer.Start(ctx, "TooManyClientRetrier") defer span.End() count := 0 diff --git a/internal/storage/ledger/accounts.go b/internal/storage/ledger/accounts.go index bf7e20665..b5d80e72c 100644 --- a/internal/storage/ledger/accounts.go +++ b/internal/storage/ledger/accounts.go @@ -331,6 +331,8 @@ func (s *Store) DeleteAccountMetadata(ctx context.Context, account, key string) return err } +// todo: since we update first balances of an accounts in the transaction process, we can avoid nested sql txs +// while upserting account and upsert them all in one shot func (s *Store) UpsertAccount(ctx context.Context, account *ledger.Account) (bool, error) { return tracing.TraceWithMetric( ctx, diff --git a/internal/storage/ledger/logs.go b/internal/storage/ledger/logs.go index 6d2ab944f..adfd29bae 100644 --- a/internal/storage/ledger/logs.go +++ b/internal/storage/ledger/logs.go @@ -47,14 +47,6 @@ func (j RawMessage) Value() (driver.Value, error) { func (s *Store) InsertLog(ctx context.Context, log *ledger.Log) error { - // We lock logs table as we need than the last log does not change until the transaction commit - if s.ledger.HasFeature(ledger.FeatureHashLogs, "SYNC") { - _, err := s.db.NewRaw(`select pg_advisory_xact_lock(?)`, s.ledger.ID).Exec(ctx) - if err != nil { - return postgres.ResolveError(err) - } - } - _, err := tracing.TraceWithMetric( ctx, "InsertLog", @@ -62,6 +54,14 @@ func (s *Store) InsertLog(ctx context.Context, log *ledger.Log) error { s.insertLogHistogram, tracing.NoResult(func(ctx context.Context) error { + // We lock logs table as we need than the last log does not change until the transaction commit + if s.ledger.HasFeature(ledger.FeatureHashLogs, "SYNC") { + _, err := s.db.NewRaw(`select pg_advisory_xact_lock(?)`, s.ledger.ID).Exec(ctx) + if err != nil { + return postgres.ResolveError(err) + } + } + payloadData, err := json.Marshal(log.Data) if err != nil { return fmt.Errorf("failed to marshal log data: %w", err) diff --git a/test/performance/benchmark_test.go b/test/performance/benchmark_test.go index 640a7c4f1..e4cf3c961 100644 --- a/test/performance/benchmark_test.go +++ b/test/performance/benchmark_test.go @@ -87,7 +87,7 @@ func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Result { report.End = time.Now() // Fetch otel metrics - rsp, err := http.Get(env.URL() + "/_metrics") + rsp, err := http.Get(env.URL() + "/_/metrics") require.NoError(b, err) ret := make(map[string]any) require.NoError(b, json.NewDecoder(rsp.Body).Decode(&ret))