From d390eabfe6e4d5aecfebaff52e876740c13729c2 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 29 Jun 2023 13:57:58 +0200 Subject: [PATCH 1/7] feat: add temporality refactor so many things for this feature... --- components/fctl/go.mod | 3 + components/fctl/go.sum | 1 + components/ledger/Taskfile.yaml | 4 +- components/ledger/benchmarks/ledger_test.go | 112 ++- components/ledger/benchmarks/main_test.go | 1 + components/ledger/cmd/container.go | 11 +- components/ledger/cmd/root.go | 4 +- components/ledger/cmd/serve.go | 13 +- components/ledger/cmd/storage.go | 10 +- components/ledger/go.mod | 5 +- components/ledger/go.sum | 7 +- components/ledger/libs/.github/dependabot.yml | 11 - .../ledger/libs/.github/workflows/main.yml | 12 - .../ledger/libs/.github/workflows/pr-open.yml | 13 - components/ledger/libs/.gitignore | 3 - .../ledger/libs/.pre-commit-config.yaml | 22 - components/ledger/libs/api/response_utils.go | 37 + .../libs/collectionutils/linked_list.go | 142 ++++ .../ledger/libs/collectionutils/slice.go | 8 + components/ledger/libs/logging/context.go | 14 + components/ledger/libs/logging/logging.go | 1 + components/ledger/libs/logging/logrus.go | 13 + components/ledger/libs/pointer/utils.go | 5 + components/ledger/libs/service/logging.go | 13 +- components/ledger/openapi.yaml | 29 +- components/ledger/pkg/analytics/analytics.go | 2 +- components/ledger/pkg/analytics/backend.go | 8 +- components/ledger/pkg/analytics/module.go | 4 +- components/ledger/pkg/api/api.go | 10 +- components/ledger/pkg/api/apierrors/errors.go | 2 +- .../pkg/api/controllers/account_controller.go | 8 - .../controllers/account_controller_test.go | 68 +- components/ledger/pkg/api/controllers/api.go | 12 +- .../ledger/pkg/api/controllers/api_test.go | 12 +- .../controllers/balance_controller_test.go | 18 +- .../api/controllers/config_controller_test.go | 2 +- .../api/controllers/ledger_controller_test.go | 20 +- .../ledger/pkg/api/controllers/query.go | 15 +- .../transaction_controller_test.go | 32 +- .../ledger/pkg/api/controllers/utils_test.go | 33 - .../pkg/api/middlewares/ledger_middleware.go | 42 +- .../pkg/api/middlewares/metrics_middleware.go | 2 +- components/ledger/pkg/api/routes/routes.go | 14 +- components/ledger/pkg/bus/aggregate.go | 23 - components/ledger/pkg/bus/message.go | 14 +- components/ledger/pkg/bus/module.go | 10 - components/ledger/pkg/bus/monitor.go | 26 +- components/ledger/pkg/bus/monitor_test.go | 4 +- components/ledger/pkg/core/account.go | 6 +- components/ledger/pkg/core/log.go | 54 +- components/ledger/pkg/core/move.go | 15 + components/ledger/pkg/core/time.go | 4 + components/ledger/pkg/core/transaction.go | 23 + .../ledger/pkg/core/transaction_test.go | 6 +- components/ledger/pkg/core/volumes.go | 69 +- .../pkg/ledger/aggregator/aggregator.go | 98 --- .../ledger/pkg/ledger/aggregator/store.go | 11 - .../ledger/pkg/ledger/command/commander.go | 33 +- .../pkg/ledger/command/commander_test.go | 51 +- .../ledger/pkg/ledger/command/context.go | 26 +- components/ledger/pkg/ledger/command/lock.go | 121 +-- components/ledger/pkg/ledger/command/store.go | 186 +---- components/ledger/pkg/ledger/ledger.go | 40 +- components/ledger/pkg/ledger/module.go | 28 +- .../ledger/pkg/ledger/monitor/monitor.go | 29 - components/ledger/pkg/ledger/query/errors.go | 11 - components/ledger/pkg/ledger/query/init.go | 143 ---- components/ledger/pkg/ledger/query/module.go | 27 - .../ledger/pkg/ledger/query/module_test.go | 87 --- components/ledger/pkg/ledger/query/monitor.go | 29 + .../ledger/pkg/ledger/query/move_buffer.go | 125 +++ .../pkg/ledger/query/move_buffer_test.go | 55 ++ .../ledger/pkg/ledger/query/projector.go | 260 +++++++ .../ledger/pkg/ledger/query/projector_test.go | 183 +++++ components/ledger/pkg/ledger/query/store.go | 31 +- components/ledger/pkg/ledger/query/worker.go | 387 ---------- .../ledger/pkg/ledger/query/worker_test.go | 268 ------- components/ledger/pkg/ledger/resolver.go | 72 +- .../pkg/ledger/utils/batching/batcher.go | 75 ++ .../ledger/pkg/ledger/utils/job/jobs.go | 134 ++++ .../ledger/pkg/ledger/utils/job/jobs_test.go | 44 ++ components/ledger/pkg/machine/vm/store.go | 2 +- .../pkg/opentelemetry/metrics/metrics.go | 64 +- components/ledger/pkg/storage/database.go | 40 + .../ledger/pkg/storage/{ => driver}/cli.go | 19 +- .../ledger/pkg/storage/{ => driver}/driver.go | 17 +- .../pkg/storage/{ => driver}/driver_test.go | 12 +- .../ledger/pkg/storage/{errors => }/errors.go | 22 +- components/ledger/pkg/storage/inmemory.go | 189 +++++ .../pkg/storage/ledgerstore/accounts.go | 466 +++-------- .../pkg/storage/ledgerstore/accounts_test.go | 246 +++--- .../pkg/storage/ledgerstore/balances.go | 265 +++---- .../pkg/storage/ledgerstore/balances_test.go | 77 +- .../ledger/pkg/storage/ledgerstore/bigint.go | 88 +++ .../ledger/pkg/storage/ledgerstore/logs.go | 314 ++++---- .../pkg/storage/ledgerstore/logs_test.go | 293 ++++++- .../pkg/storage/ledgerstore/logs_worker.go | 43 +- .../pkg/storage/ledgerstore/main_test.go | 57 +- .../ledgerstore/migrates/0-init-schema/any.go | 286 +------ .../migrates/0-init-schema/postgres.sql | 142 ++-- .../1-optimized-accounts-volumes/postgres.sql | 12 - .../pkg/storage/ledgerstore/migration.go | 2 +- .../storage/ledgerstore/pagination_column.go | 7 +- .../ledgerstore/pagination_column_test.go | 107 ++- .../storage/ledgerstore/pagination_offset.go | 25 +- .../ledgerstore/pagination_offset_test.go | 13 +- .../ledger/pkg/storage/ledgerstore/store.go | 86 +-- .../pkg/storage/ledgerstore/store_test.go | 330 +------- .../pkg/storage/ledgerstore/transactions.go | 628 +++++++++------ .../storage/ledgerstore/transactions_test.go | 730 ++++++++++++++++-- .../ledger/pkg/storage/ledgerstore/utils.go | 5 - .../ledger/pkg/storage/ledgerstore/volumes.go | 113 +-- .../pkg/storage/ledgerstore/volumes_test.go | 116 +-- .../pkg/storage/migrations/migrations.go | 39 +- .../pkg/storage/migrations/migrations_test.go | 14 +- .../storage/opentelemetry/metrics/metrics.go | 27 - .../ledger/pkg/storage/{schema => }/schema.go | 78 +- .../storage.go | 11 +- .../pkg/storage/systemstore/configuration.go | 2 +- .../ledger/pkg/storage/systemstore/ledgers.go | 2 +- .../ledger/pkg/storage/systemstore/store.go | 11 +- components/ledger/pkg/storage/tx.go | 23 + .../ledger/pkg/storage/{utils => }/utils.go | 12 +- .../benthos/streams/ledger_ingestion.yaml | 55 +- .../v1/ledger/COMMITTED_TRANSACTIONS.yaml | 68 -- .../v1/ledger/REVERTED_TRANSACTION.yaml | 56 -- libs/go-libs/api/response_utils.go | 37 + libs/go-libs/collectionutils/linked_list.go | 142 ++++ libs/go-libs/collectionutils/slice.go | 8 + libs/go-libs/logging/context.go | 14 + libs/go-libs/logging/logging.go | 1 + libs/go-libs/logging/logrus.go | 13 + libs/go-libs/pointer/utils.go | 5 + libs/go-libs/service/logging.go | 13 +- tests/integration/go.mod | 1 + tests/integration/go.sum | 2 + tests/integration/internal/bootstrap.go | 8 +- tests/integration/suite/ledger-balances.go | 42 +- .../suite/ledger-create-transaction.go | 31 - .../suite/ledger-list-count-accounts.go | 62 +- 140 files changed, 4603 insertions(+), 4596 deletions(-) create mode 100644 components/ledger/libs/api/response_utils.go create mode 100644 components/ledger/libs/collectionutils/linked_list.go create mode 100644 components/ledger/libs/pointer/utils.go delete mode 100644 components/ledger/pkg/bus/aggregate.go delete mode 100644 components/ledger/pkg/bus/module.go create mode 100644 components/ledger/pkg/core/move.go delete mode 100644 components/ledger/pkg/ledger/aggregator/aggregator.go delete mode 100644 components/ledger/pkg/ledger/aggregator/store.go delete mode 100644 components/ledger/pkg/ledger/monitor/monitor.go delete mode 100644 components/ledger/pkg/ledger/query/errors.go delete mode 100644 components/ledger/pkg/ledger/query/init.go delete mode 100644 components/ledger/pkg/ledger/query/module.go delete mode 100644 components/ledger/pkg/ledger/query/module_test.go create mode 100644 components/ledger/pkg/ledger/query/monitor.go create mode 100644 components/ledger/pkg/ledger/query/move_buffer.go create mode 100644 components/ledger/pkg/ledger/query/move_buffer_test.go create mode 100644 components/ledger/pkg/ledger/query/projector.go create mode 100644 components/ledger/pkg/ledger/query/projector_test.go delete mode 100644 components/ledger/pkg/ledger/query/worker.go delete mode 100644 components/ledger/pkg/ledger/query/worker_test.go create mode 100644 components/ledger/pkg/ledger/utils/batching/batcher.go create mode 100644 components/ledger/pkg/ledger/utils/job/jobs.go create mode 100644 components/ledger/pkg/ledger/utils/job/jobs_test.go create mode 100644 components/ledger/pkg/storage/database.go rename components/ledger/pkg/storage/{ => driver}/cli.go (83%) rename components/ledger/pkg/storage/{ => driver}/driver.go (88%) rename components/ledger/pkg/storage/{ => driver}/driver_test.go (76%) rename components/ledger/pkg/storage/{errors => }/errors.go (64%) create mode 100644 components/ledger/pkg/storage/inmemory.go create mode 100644 components/ledger/pkg/storage/ledgerstore/bigint.go delete mode 100644 components/ledger/pkg/storage/ledgerstore/migrates/1-optimized-accounts-volumes/postgres.sql delete mode 100644 components/ledger/pkg/storage/ledgerstore/utils.go delete mode 100644 components/ledger/pkg/storage/opentelemetry/metrics/metrics.go rename components/ledger/pkg/storage/{schema => }/schema.go (55%) rename components/ledger/pkg/storage/{sqlstoragetesting => storagetesting}/storage.go (64%) create mode 100644 components/ledger/pkg/storage/tx.go rename components/ledger/pkg/storage/{utils => }/utils.go (78%) create mode 100644 libs/go-libs/api/response_utils.go create mode 100644 libs/go-libs/collectionutils/linked_list.go create mode 100644 libs/go-libs/pointer/utils.go diff --git a/components/fctl/go.mod b/components/fctl/go.mod index 5183994d29..f3b17e20d2 100644 --- a/components/fctl/go.mod +++ b/components/fctl/go.mod @@ -30,6 +30,7 @@ require ( atomicgo.dev/keyboard v0.2.9 // indirect github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.14.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/uuid v1.3.0 // indirect @@ -47,11 +48,13 @@ require ( github.com/mattn/go-tty v0.0.4 // indirect github.com/muhlemmer/gu v0.3.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/segmentio/backo-go v1.0.1 // indirect github.com/sergi/go-diff v1.3.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.8.4 // indirect github.com/talal/go-bits v0.0.0-20200204154716-071e9f3e66e1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/crypto v0.10.0 // indirect diff --git a/components/fctl/go.sum b/components/fctl/go.sum index 6ac829d0f3..9a10ab5f33 100644 --- a/components/fctl/go.sum +++ b/components/fctl/go.sum @@ -140,6 +140,7 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/talal/go-bits v0.0.0-20200204154716-071e9f3e66e1 h1:sDCJkxkdgPIDUKcOemdIsP2AnUjtLTVSlUDkAnf3fB4= github.com/talal/go-bits v0.0.0-20200204154716-071e9f3e66e1/go.mod h1:IaWL8TVo0gKkfVx+4RbcRkzp6FoeMqEtD88+5aCRwyY= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= diff --git a/components/ledger/Taskfile.yaml b/components/ledger/Taskfile.yaml index 5b0c026ba6..58f0c8dc3c 100644 --- a/components/ledger/Taskfile.yaml +++ b/components/ledger/Taskfile.yaml @@ -40,17 +40,19 @@ tasks: cmds: - mkdir -p {{.BENCH_RESULTS_DIR}} - > - go test -run BenchmarkParallelWrites -bench=. + go test -run BenchmarkParallelWrites -bench=. {{if eq .VERBOSE "true"}}-v{{end}} -test.benchmem -timeout 1h -memprofile {{.BENCH_RESULTS_DIR}}/{{.BRANCH}}-memprofile-{{if eq .ASYNC "true"}}async{{else}}sync{{end}}.out -cpuprofile {{.BENCH_RESULTS_DIR}}/{{.BRANCH}}-profile-{{if eq .ASYNC "true"}}async{{else}}sync{{end}}.out -benchtime={{if .DURATION}}{{.DURATION}}{{else}}15s{{end}} + {{if eq .RACE "true"}}-race{{end}} -count={{if .COUNT}}{{.COUNT}}{{else}}10{{end}} ./benchmarks | tee {{.BENCH_RESULTS_DIR}}/{{.BRANCH}}-{{if eq .ASYNC "true"}}async{{else}}sync{{end}}.stats env: ASYNC: "{{.ASYNC}}" GOMEMLIMIT: 1GiB GOMAXPROCS: 2 + VERBOSE: false # GOGC: "1000" # https://dave.cheney.net/tag/gogc CGO_ENABLED: 0 # GODEBUG: gctrace=1 #,gcpacertrace=1 diff --git a/components/ledger/benchmarks/ledger_test.go b/components/ledger/benchmarks/ledger_test.go index 24adf4dcc6..f761e9012d 100644 --- a/components/ledger/benchmarks/ledger_test.go +++ b/components/ledger/benchmarks/ledger_test.go @@ -2,7 +2,6 @@ package benchmarks import ( "bytes" - "context" "encoding/json" "fmt" "net/http" @@ -10,6 +9,7 @@ import ( "net/url" "os" "runtime" + "sync" "testing" "time" @@ -18,7 +18,9 @@ import ( "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/ledger" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage/sqlstoragetesting" + "github.com/formancehq/ledger/pkg/storage/storagetesting" + "github.com/formancehq/stack/libs/go-libs/api" + "github.com/formancehq/stack/libs/go-libs/logging" "github.com/google/uuid" "github.com/stretchr/testify/require" "go.uber.org/atomic" @@ -26,18 +28,22 @@ import ( func BenchmarkParallelWrites(b *testing.B) { - driver := sqlstoragetesting.StorageDriver(b) - resolver := ledger.NewResolver(driver) + ctx := logging.TestingContext() + + driver := storagetesting.StorageDriver(b) + resolver := ledger.NewResolver(driver, ledger.WithLogger(logging.FromContext(ctx))) b.Cleanup(func() { - require.NoError(b, resolver.CloseLedgers(context.Background())) + require.NoError(b, resolver.CloseLedgers(ctx)) }) ledgerName := uuid.NewString() backend := controllers.NewDefaultBackend(driver, "latest", resolver) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) - srv := httptest.NewServer(router) - defer srv.Close() + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := logging.ContextWithLogger(r.Context(), logging.FromContext(ctx)) + router.ServeHTTP(w, r.WithContext(ctx)) + }) totalDuration := atomic.Int64{} b.SetParallelism(1000) @@ -45,45 +51,99 @@ func BenchmarkParallelWrites(b *testing.B) { b.ResetTimer() startOfBench := time.Now() counter := atomic.NewInt64(0) + longestTxLock := sync.Mutex{} + longestTransactionID := uint64(0) + longestTransactionDuration := time.Duration(0) b.RunParallel(func(pb *testing.PB) { buf := bytes.NewBufferString("") for pb.Next() { buf.Reset() + id := counter.Add(1) - err := json.NewEncoder(buf).Encode(controllers.PostTransactionRequest{ - Script: controllers.Script{ - Script: core.Script{ - Plain: ` - vars { - account $account - } - - send [USD/2 100] ( - source = @world - destination = $account - )`, - }, - Vars: map[string]any{ - "account": fmt.Sprintf("accounts:%d", counter.Add(1)), - }, + //script := controllers.Script{ + // Script: core.Script{ + // Plain: fmt.Sprintf(` + // vars { + // account $account + // } + // + // send [USD/2 100] ( + // source = @world:%d allowing unbounded overdraft + // destination = $account + // )`, counter.Load()%100), + // }, + // Vars: map[string]any{ + // "account": fmt.Sprintf("accounts:%d", counter.Add(1)), + // }, + //} + + script := controllers.Script{ + Script: core.Script{ + Plain: `vars { + account $account +} + +send [USD/2 100] ( + source = @world + destination = $account +)`, + }, + Vars: map[string]any{ + "account": fmt.Sprintf("accounts:%d", id), }, + } + + // script := controllers.Script{ + // Script: core.Script{ + // Plain: `vars { + // account $account + // account $src + //} + // + //send [USD/2 100] ( + // source = $src allowing unbounded overdraft + // destination = $account + //)`, + // }, + // Vars: map[string]any{ + // "src": fmt.Sprintf("world:%d", id), + // "account": fmt.Sprintf("accounts:%d", id), + // }, + // } + + err := json.NewEncoder(buf).Encode(controllers.PostTransactionRequest{ + Script: script, }) require.NoError(b, err) + //ctx, _ := context.WithDeadline(ctx, time.Now().Add(10*time.Second)) + req := httptest.NewRequest("POST", "/"+ledgerName+"/transactions", buf) + req = req.WithContext(ctx) req.URL.RawQuery = url.Values{ "async": []string{os.Getenv("ASYNC")}, }.Encode() rsp := httptest.NewRecorder() now := time.Now() - router.ServeHTTP(rsp, req) - totalDuration.Add(time.Since(now).Milliseconds()) + handler.ServeHTTP(rsp, req) + latency := time.Since(now).Milliseconds() + totalDuration.Add(latency) require.Equal(b, http.StatusOK, rsp.Code) + tx, _ := api.DecodeSingleResponse[core.Transaction](b, rsp.Body) + + longestTxLock.Lock() + if time.Millisecond*time.Duration(latency) > longestTransactionDuration { + longestTransactionID = tx.ID + longestTransactionDuration = time.Duration(latency) * time.Millisecond + } + longestTxLock.Unlock() } }) + b.StopTimer() + b.Logf("Longest transaction: %d (%s)", longestTransactionID, longestTransactionDuration.String()) b.ReportMetric((float64(time.Duration(b.N))/float64(time.Since(startOfBench)))*float64(time.Second), "t/s") b.ReportMetric(float64(totalDuration.Load()/int64(b.N)), "ms/transaction") runtime.GC() diff --git a/components/ledger/benchmarks/main_test.go b/components/ledger/benchmarks/main_test.go index 3f2192ea80..72d31abb8c 100644 --- a/components/ledger/benchmarks/main_test.go +++ b/components/ledger/benchmarks/main_test.go @@ -11,6 +11,7 @@ import ( ) func TestMain(m *testing.M) { + if err := pgtesting.CreatePostgresServer(pgtesting.WithDockerHostConfigOption(func(hostConfig *docker.HostConfig) { hostConfig.CPUCount = 2 })); err != nil { diff --git a/components/ledger/cmd/container.go b/components/ledger/cmd/container.go index 2a8548d8c3..c374dbf9bf 100644 --- a/components/ledger/cmd/container.go +++ b/components/ledger/cmd/container.go @@ -5,9 +5,8 @@ import ( "github.com/formancehq/ledger/cmd/internal" "github.com/formancehq/ledger/pkg/api" - "github.com/formancehq/ledger/pkg/bus" "github.com/formancehq/ledger/pkg/ledger" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/stack/libs/go-libs/otlp/otlpmetrics" "github.com/formancehq/stack/libs/go-libs/otlp/otlptraces" "github.com/formancehq/stack/libs/go-libs/publish" @@ -25,26 +24,22 @@ func resolveOptions(output io.Writer, userOptions ...fx.Option) []fx.Option { v := viper.GetViper() debug := v.GetBool(service.DebugFlag) if debug { - storage.InstrumentalizeSQLDriver() + driver.InstrumentalizeSQLDriver() } options = append(options, publish.CLIPublisherModule(v, ServiceName), - bus.LedgerMonitorModule(), otlptraces.CLITracesModule(v), otlpmetrics.CLIMetricsModule(v), api.Module(api.Config{ Version: Version, }), - storage.CLIDriverModule(v, output, debug), + driver.CLIModule(v, output, debug), internal.NewAnalyticsModule(v, Version), ledger.Module(ledger.Configuration{ NumscriptCache: ledger.NumscriptCacheConfiguration{ MaxCount: v.GetInt(numscriptCacheMaxCount), }, - Query: ledger.QueryConfiguration{ - LimitReadLogs: v.GetInt(queryLimitReadLogsFlag), - }, }), ) diff --git a/components/ledger/cmd/root.go b/components/ledger/cmd/root.go index 1c429a7ed4..e95bb4b63f 100644 --- a/components/ledger/cmd/root.go +++ b/components/ledger/cmd/root.go @@ -5,7 +5,7 @@ import ( "os" "github.com/formancehq/ledger/cmd/internal" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" initschema "github.com/formancehq/ledger/pkg/storage/ledgerstore/migrates/0-init-schema" "github.com/formancehq/stack/libs/go-libs/otlp/otlpmetrics" "github.com/formancehq/stack/libs/go-libs/otlp/otlptraces" @@ -61,7 +61,7 @@ func NewRootCommand() *cobra.Command { otlptraces.InitOTLPTracesFlags(root.PersistentFlags()) internal.InitAnalyticsFlags(root, DefaultSegmentWriteKey) publish.InitCLIFlags(root) - storage.InitCLIFlags(root) + driver.InitCLIFlags(root) initschema.InitMigrationConfigCLIFlags(root.PersistentFlags()) if err := viper.BindPFlags(root.PersistentFlags()); err != nil { diff --git a/components/ledger/cmd/serve.go b/components/ledger/cmd/serve.go index fa3dec0d64..275ca89c43 100644 --- a/components/ledger/cmd/serve.go +++ b/components/ledger/cmd/serve.go @@ -1,9 +1,12 @@ package cmd import ( + "net/http" + "github.com/formancehq/ledger/pkg/api/middlewares" "github.com/formancehq/stack/libs/go-libs/ballast" "github.com/formancehq/stack/libs/go-libs/httpserver" + "github.com/formancehq/stack/libs/go-libs/logging" app "github.com/formancehq/stack/libs/go-libs/service" "github.com/go-chi/chi/v5" "github.com/spf13/cobra" @@ -12,7 +15,6 @@ import ( ) const ( - queryLimitReadLogsFlag = "query-limit-read-logs" ballastSizeInBytesFlag = "ballast-size" numscriptCacheMaxCount = "numscript-cache-max-count" ) @@ -24,10 +26,16 @@ func NewServe() *cobra.Command { return app.New(cmd.OutOrStdout(), resolveOptions( cmd.OutOrStdout(), ballast.Module(viper.GetUint(ballastSizeInBytesFlag)), - fx.Invoke(func(lc fx.Lifecycle, h chi.Router) { + fx.Invoke(func(lc fx.Lifecycle, h chi.Router, logger logging.Logger) { if viper.GetBool(app.DebugFlag) { wrappedRouter := chi.NewRouter() + wrappedRouter.Use(func(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r = r.WithContext(logging.ContextWithLogger(r.Context(), logger)) + handler.ServeHTTP(w, r) + }) + }) wrappedRouter.Use(middlewares.Log()) wrappedRouter.Mount("/", h) h = wrappedRouter @@ -38,7 +46,6 @@ func NewServe() *cobra.Command { )...).Run(cmd.Context()) }, } - cmd.Flags().Int(queryLimitReadLogsFlag, 10000, "Query limit read logs") cmd.Flags().Uint(ballastSizeInBytesFlag, 0, "Ballast size in bytes, default to 0") cmd.Flags().Int(numscriptCacheMaxCount, 1024, "Numscript cache max count") return cmd diff --git a/components/ledger/cmd/storage.go b/components/ledger/cmd/storage.go index e66b2083ac..90186dfa0b 100644 --- a/components/ledger/cmd/storage.go +++ b/components/ledger/cmd/storage.go @@ -4,7 +4,7 @@ import ( "context" "errors" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/service" "github.com/spf13/cobra" @@ -26,7 +26,7 @@ func NewStorageInit() *cobra.Command { cmd.OutOrStdout(), resolveOptions( cmd.OutOrStdout(), - fx.Invoke(func(storageDriver *storage.Driver, lc fx.Lifecycle) { + fx.Invoke(func(storageDriver *driver.Driver, lc fx.Lifecycle) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { name := viper.GetString("name") @@ -71,7 +71,7 @@ func NewStorageList() *cobra.Command { app := service.New(cmd.OutOrStdout(), resolveOptions( cmd.OutOrStdout(), - fx.Invoke(func(storageDriver *storage.Driver, lc fx.Lifecycle) { + fx.Invoke(func(storageDriver *driver.Driver, lc fx.Lifecycle) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { ledgers, err := storageDriver.GetSystemStore().ListLedgers(ctx) @@ -103,7 +103,7 @@ func NewStorageUpgrade() *cobra.Command { app := service.New(cmd.OutOrStdout(), resolveOptions( cmd.OutOrStdout(), - fx.Invoke(func(storageDriver *storage.Driver, lc fx.Lifecycle) { + fx.Invoke(func(storageDriver *driver.Driver, lc fx.Lifecycle) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { name := args[0] @@ -140,7 +140,7 @@ func NewStorageDelete() *cobra.Command { cmd.OutOrStdout(), resolveOptions( cmd.OutOrStdout(), - fx.Invoke(func(storageDriver *storage.Driver, lc fx.Lifecycle) { + fx.Invoke(func(storageDriver *driver.Driver, lc fx.Lifecycle) { lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { name := args[0] diff --git a/components/ledger/go.mod b/components/ledger/go.mod index c9cad240e5..a928032f24 100644 --- a/components/ledger/go.mod +++ b/components/ledger/go.mod @@ -5,12 +5,14 @@ go 1.19 require ( github.com/Masterminds/semver/v3 v3.2.0 github.com/ThreeDotsLabs/watermill v1.2.0 + github.com/alitto/pond v1.8.3 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 github.com/bluele/gcache v0.0.2 github.com/formancehq/stack/libs/go-libs v0.0.0-20230517212829-71aaaacfd130 github.com/go-chi/chi/v5 v5.0.8 github.com/go-chi/cors v1.2.1 github.com/golang/mock v1.4.4 + github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 github.com/jackc/pgx/v5 v5.3.0 github.com/lib/pq v1.10.7 @@ -30,13 +32,13 @@ require ( github.com/uptrace/bun/dialect/pgdialect v1.1.12 github.com/uptrace/bun/extra/bunbig v1.1.12 github.com/uptrace/bun/extra/bundebug v1.1.12 + github.com/uptrace/bun/extra/bunotel v1.1.12 go.nhat.io/otelsql v0.9.0 go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/metric v0.37.0 go.opentelemetry.io/otel/trace v1.14.0 go.uber.org/atomic v1.10.0 go.uber.org/fx v1.19.2 - golang.org/x/sync v0.1.0 gopkg.in/segmentio/analytics-go.v3 v3.1.0 ) @@ -116,6 +118,7 @@ require ( github.com/tklauser/numcpus v0.6.0 // indirect github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect github.com/uptrace/opentelemetry-go-extra/otellogrus v0.1.21 // indirect + github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.21 // indirect github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.21 // indirect github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect diff --git a/components/ledger/go.sum b/components/ledger/go.sum index 52305270fb..010590e4db 100644 --- a/components/ledger/go.sum +++ b/components/ledger/go.sum @@ -64,6 +64,8 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs= +github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= @@ -452,8 +454,12 @@ github.com/uptrace/bun/extra/bunbig v1.1.12 h1:Dg/bCwO30zmOb/KwctsBVIoz+vOBBcKbB github.com/uptrace/bun/extra/bunbig v1.1.12/go.mod h1:EU3WwCvNYFpJjCUI0EKTPVRlYW8kAXy6nUbhOlQl5NE= github.com/uptrace/bun/extra/bundebug v1.1.12 h1:y8nrHvo7TUCR91kXngWuF7Bk0E1nCTsWzYL1CDEriTo= github.com/uptrace/bun/extra/bundebug v1.1.12/go.mod h1:psjCrCMf5JaAyivW/A8MDBW5MwIy/jZFBCkIaBgabtM= +github.com/uptrace/bun/extra/bunotel v1.1.12 h1:uWPU75j9dYGXMRC9jF0ASlndZZAcngoqZagH4w3kn54= +github.com/uptrace/bun/extra/bunotel v1.1.12/go.mod h1:QfszJGLzNaTTGvvg17cEEUyEwxXq2NJ7sRvrPYvYSIU= github.com/uptrace/opentelemetry-go-extra/otellogrus v0.1.21 h1:OXsouNDvuET5o1A4uvoCnAXuuNke8JlfZWceciyUlC8= github.com/uptrace/opentelemetry-go-extra/otellogrus v0.1.21/go.mod h1:Xm3wlRGm5xzdAGPOvqydXPiGj0Da1q0OlUNm7Utoda4= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.21 h1:iHkIlTU2P3xbSbVJbAiHL9IT+ekYV5empheF+652yeQ= +github.com/uptrace/opentelemetry-go-extra/otelsql v0.1.21/go.mod h1:hiCFa1UeZITKXi8lhu2qwOD5LHXjdGMCUIQHbybxoF0= github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.21 h1:HCqo51kNF8wxDMDhxcN5S6DlfZXigMtptRpkvjBCeVc= github.com/uptrace/opentelemetry-go-extra/otelutil v0.1.21/go.mod h1:2MNqrUmDrt5E0glMuoJI/9FyGVpBKo1FqjSH60UOZFg= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= @@ -663,7 +669,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/components/ledger/libs/.github/dependabot.yml b/components/ledger/libs/.github/dependabot.yml index 83a8ab4395..e69de29bb2 100644 --- a/components/ledger/libs/.github/dependabot.yml +++ b/components/ledger/libs/.github/dependabot.yml @@ -1,11 +0,0 @@ -version: 2 -updates: - - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - - package-ecosystem: "gomod" - directory: "/" - schedule: - interval: "weekly" diff --git a/components/ledger/libs/.github/workflows/main.yml b/components/ledger/libs/.github/workflows/main.yml index 413db79743..e69de29bb2 100644 --- a/components/ledger/libs/.github/workflows/main.yml +++ b/components/ledger/libs/.github/workflows/main.yml @@ -1,12 +0,0 @@ -on: - push: - branches: - - main - -name: Main -jobs: - lint: - uses: formancehq/gh-workflows/.github/workflows/golang-lint.yml@main - - test: - uses: formancehq/gh-workflows/.github/workflows/golang-test.yml@main diff --git a/components/ledger/libs/.github/workflows/pr-open.yml b/components/ledger/libs/.github/workflows/pr-open.yml index 90eda008e3..e69de29bb2 100644 --- a/components/ledger/libs/.github/workflows/pr-open.yml +++ b/components/ledger/libs/.github/workflows/pr-open.yml @@ -1,13 +0,0 @@ -name: Pull Request - Open -on: - pull_request: - types: [assigned, opened, synchronize, reopened] -jobs: - pr-style: - uses: formancehq/gh-workflows/.github/workflows/pr-style.yml@main - - lint: - uses: formancehq/gh-workflows/.github/workflows/golang-lint.yml@main - - test: - uses: formancehq/gh-workflows/.github/workflows/golang-test.yml@main diff --git a/components/ledger/libs/.gitignore b/components/ledger/libs/.gitignore index 2a9b1e54c7..e69de29bb2 100644 --- a/components/ledger/libs/.gitignore +++ b/components/ledger/libs/.gitignore @@ -1,3 +0,0 @@ -.idea -vendor -coverage.* diff --git a/components/ledger/libs/.pre-commit-config.yaml b/components/ledger/libs/.pre-commit-config.yaml index a4c584c911..e69de29bb2 100644 --- a/components/ledger/libs/.pre-commit-config.yaml +++ b/components/ledger/libs/.pre-commit-config.yaml @@ -1,22 +0,0 @@ -exclude: client -fail_fast: true -repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 - hooks: - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-yaml - exclude: .cloud - - id: check-added-large-files -- repo: https://github.com/formancehq/pre-commit-hooks - rev: dd079f7c30ad72446d615f55a000d4f875e79633 - hooks: - - id: gogenerate - files: swagger.yaml - - id: gomodtidy - - id: goimports - - id: gofmt - - id: golangci-lint - - id: gotests - - id: commitlint diff --git a/components/ledger/libs/api/response_utils.go b/components/ledger/libs/api/response_utils.go new file mode 100644 index 0000000000..f861f27caf --- /dev/null +++ b/components/ledger/libs/api/response_utils.go @@ -0,0 +1,37 @@ +package api + +import ( + "bytes" + "encoding/json" + "io" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Encode(t require.TestingT, v interface{}) []byte { + data, err := json.Marshal(v) + assert.NoError(t, err) + return data +} + +func Buffer(t require.TestingT, v interface{}) *bytes.Buffer { + return bytes.NewBuffer(Encode(t, v)) +} + +func Decode(t require.TestingT, reader io.Reader, v interface{}) { + err := json.NewDecoder(reader).Decode(v) + require.NoError(t, err) +} + +func DecodeSingleResponse[T any](t require.TestingT, reader io.Reader) (T, bool) { + res := BaseResponse[T]{} + Decode(t, reader, &res) + return *res.Data, true +} + +func DecodeCursorResponse[T any](t require.TestingT, reader io.Reader) *Cursor[T] { + res := BaseResponse[T]{} + Decode(t, reader, &res) + return res.Cursor +} diff --git a/components/ledger/libs/collectionutils/linked_list.go b/components/ledger/libs/collectionutils/linked_list.go new file mode 100644 index 0000000000..b6ecb8ff97 --- /dev/null +++ b/components/ledger/libs/collectionutils/linked_list.go @@ -0,0 +1,142 @@ +package collectionutils + +import ( + "sync" +) + +type LinkedListNode[T any] struct { + object T + list *LinkedList[T] + previousNode, nextNode *LinkedListNode[T] +} + +func (n *LinkedListNode[T]) Next() *LinkedListNode[T] { + return n.nextNode +} + +func (n *LinkedListNode[T]) Value() T { + return n.object +} + +func (n *LinkedListNode[T]) Remove() { + if n.previousNode != nil { + n.previousNode.nextNode = n.nextNode + } + if n.nextNode != nil { + n.nextNode.previousNode = n.previousNode + } + if n == n.list.firstNode { + n.list.firstNode = n.nextNode + } + if n == n.list.lastNode { + n.list.lastNode = n.previousNode + } +} + +type LinkedList[T any] struct { + mu sync.Mutex + firstNode, lastNode *LinkedListNode[T] +} + +func (r *LinkedList[T]) Append(objects ...T) { + r.mu.Lock() + defer r.mu.Unlock() + + for _, object := range objects { + if r.firstNode == nil { + r.firstNode = &LinkedListNode[T]{ + object: object, + list: r, + } + r.lastNode = r.firstNode + continue + } + r.lastNode = &LinkedListNode[T]{ + object: object, + previousNode: r.lastNode, + list: r, + } + r.lastNode.previousNode.nextNode = r.lastNode + } +} + +func (r *LinkedList[T]) RemoveFirst(cmp func(T) bool) *LinkedListNode[T] { + r.mu.Lock() + defer r.mu.Unlock() + + node := r.firstNode + for node != nil { + if cmp(node.object) { + node.Remove() + return node + } + node = node.nextNode + } + + return nil +} + +func (r *LinkedList[T]) RemoveValue(t T) *LinkedListNode[T] { + return r.RemoveFirst(func(t2 T) bool { + return (any)(t) == (any)(t2) + }) +} + +func (r *LinkedList[T]) TakeFirst() T { + var t T + if r.firstNode == nil { + return t + } + ret := r.firstNode.object + if r.firstNode.nextNode == nil { + r.firstNode = nil + } else { + r.firstNode = r.firstNode.nextNode + r.firstNode.previousNode = nil + } + return ret +} + +func (r *LinkedList[T]) Length() int { + r.mu.Lock() + defer r.mu.Unlock() + + count := 0 + + node := r.firstNode + for node != nil { + count++ + node = node.nextNode + } + + return count +} + +func (r *LinkedList[T]) ForEach(f func(t T)) { + r.mu.Lock() + defer r.mu.Unlock() + + node := r.firstNode + for node != nil { + f(node.object) + node = node.nextNode + } +} + +func (r *LinkedList[T]) Slice() []T { + ret := make([]T, 0) + node := r.firstNode + for node != nil { + ret = append(ret, node.object) + node = node.nextNode + } + return ret +} + +func (r *LinkedList[T]) FirstNode() *LinkedListNode[T] { + return r.firstNode +} + +func NewLinkedList[T any]() *LinkedList[T] { + return &LinkedList[T]{} +} diff --git a/components/ledger/libs/collectionutils/slice.go b/components/ledger/libs/collectionutils/slice.go index 83cb1e8c44..be5f4a55d8 100644 --- a/components/ledger/libs/collectionutils/slice.go +++ b/components/ledger/libs/collectionutils/slice.go @@ -22,6 +22,14 @@ func Filter[TYPE any](input []TYPE, filter func(TYPE) bool) []TYPE { return ret } +func Flatten[TYPE any](input [][]TYPE) []TYPE { + ret := make([]TYPE, 0) + for _, types := range input { + ret = append(ret, types...) + } + return ret +} + func First[TYPE any](input []TYPE, filter func(TYPE) bool) TYPE { var zero TYPE ret := Filter(input, filter) diff --git a/components/ledger/libs/logging/context.go b/components/ledger/libs/logging/context.go index 7e8713231e..fcf9fb20c3 100644 --- a/components/ledger/libs/logging/context.go +++ b/components/ledger/libs/logging/context.go @@ -21,3 +21,17 @@ func FromContext(ctx context.Context) Logger { func ContextWithLogger(ctx context.Context, l Logger) context.Context { return context.WithValue(ctx, loggerKey, l) } + +func ContextWithFields(ctx context.Context, fields map[string]any) context.Context { + return ContextWithLogger(ctx, FromContext(ctx).WithFields(fields)) +} + +func ContextWithField(ctx context.Context, key string, value any) context.Context { + return ContextWithLogger(ctx, FromContext(ctx).WithFields(map[string]any{ + key: value, + })) +} + +func TestingContext() context.Context { + return ContextWithLogger(context.Background(), Testing()) +} diff --git a/components/ledger/libs/logging/logging.go b/components/ledger/libs/logging/logging.go index 495d9e5248..f161d462d9 100644 --- a/components/ledger/libs/logging/logging.go +++ b/components/ledger/libs/logging/logging.go @@ -10,6 +10,7 @@ type Logger interface { Info(args ...any) Error(args ...any) WithFields(map[string]any) Logger + WithField(key string, value any) Logger WithContext(ctx context.Context) Logger } diff --git a/components/ledger/libs/logging/logrus.go b/components/ledger/libs/logging/logrus.go index de01e87f8a..2ac5356f19 100644 --- a/components/ledger/libs/logging/logrus.go +++ b/components/ledger/libs/logging/logrus.go @@ -19,6 +19,7 @@ type logrusLogger struct { Errorf(format string, args ...any) Error(args ...any) WithFields(fields logrus.Fields) *logrus.Entry + WithField(key string, value any) *logrus.Entry WithContext(ctx context.Context) *logrus.Entry } } @@ -53,6 +54,12 @@ func (l *logrusLogger) WithFields(fields map[string]any) Logger { } } +func (l *logrusLogger) WithField(key string, value any) Logger { + return l.WithFields(map[string]any{ + key: value, + }) +} + var _ Logger = &logrusLogger{} func NewLogrus(logger *logrus.Logger) *logrusLogger { @@ -69,5 +76,11 @@ func Testing() *logrusLogger { logger.SetOutput(os.Stdout) logger.SetLevel(logrus.DebugLevel) } + + textFormatter := new(logrus.TextFormatter) + textFormatter.TimestampFormat = "15-01-2018 15:04:05.000000" + textFormatter.FullTimestamp = true + logger.SetFormatter(textFormatter) + return NewLogrus(logger) } diff --git a/components/ledger/libs/pointer/utils.go b/components/ledger/libs/pointer/utils.go new file mode 100644 index 0000000000..837c3991b0 --- /dev/null +++ b/components/ledger/libs/pointer/utils.go @@ -0,0 +1,5 @@ +package pointer + +func For[T any](t T) *T { + return &t +} diff --git a/components/ledger/libs/service/logging.go b/components/ledger/libs/service/logging.go index 75bd0c7b30..5d308c80b7 100644 --- a/components/ledger/libs/service/logging.go +++ b/components/ledger/libs/service/logging.go @@ -14,15 +14,24 @@ import ( func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonFormattingLog bool) context.Context { l := logrus.New() l.SetOutput(w) - if debug { l.Level = logrus.DebugLevel } + var formatter logrus.Formatter if jsonFormattingLog { - l.SetFormatter(&logrus.JSONFormatter{}) + jsonFormatter := &logrus.JSONFormatter{} + jsonFormatter.TimestampFormat = "15-01-2018 15:04:05.000000" + formatter = jsonFormatter + } else { + textFormatter := new(logrus.TextFormatter) + textFormatter.TimestampFormat = "15-01-2018 15:04:05.000000" + textFormatter.FullTimestamp = true + formatter = textFormatter } + l.SetFormatter(formatter) + if viper.GetBool(otlptraces.OtelTracesFlag) { l.AddHook(otellogrus.NewHook(otellogrus.WithLevels( logrus.PanicLevel, diff --git a/components/ledger/openapi.yaml b/components/ledger/openapi.yaml index 79a804319c..91054d2337 100644 --- a/components/ledger/openapi.yaml +++ b/components/ledger/openapi.yaml @@ -141,21 +141,6 @@ paths: additionalProperties: type: string example: { admin: "true" } - - name: balance - in: query - description: Filter accounts by their balance (default operator is gte) - schema: - type: integer - format: bigint - example: 2400 - - name: balanceOperator - in: query - description: | - Operator used for the filtering of balances can be greater than/equal, less than/equal, greater than, less than, equal or not. - schema: - type: string - enum: [gte, lte, gt, lt, e, ne] - example: gte - name: cursor in: query description: | @@ -1070,14 +1055,14 @@ components: type: object additionalProperties: type: integer - format: bigint + format: int64 minimum: 0 example: { COIN: { input: 100, output: 0 } } balances: type: object additionalProperties: type: integer - format: bigint + format: int64 example: COIN: 100 @@ -1097,7 +1082,7 @@ components: type: object additionalProperties: type: integer - format: bigint + format: int64 example: USD: 100 EUR: 12 @@ -1107,7 +1092,7 @@ components: properties: amount: type: integer - format: bigint + format: int64 minimum: 0 example: 100 asset: @@ -1298,13 +1283,13 @@ components: properties: input: type: integer - format: bigint + format: int64 output: type: integer - format: bigint + format: int64 balance: type: integer - format: bigint + format: int64 required: - input - output diff --git a/components/ledger/pkg/analytics/analytics.go b/components/ledger/pkg/analytics/analytics.go index 00250e0c82..3d37000b3e 100644 --- a/components/ledger/pkg/analytics/analytics.go +++ b/components/ledger/pkg/analytics/analytics.go @@ -8,7 +8,7 @@ import ( "time" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/pbnjay/memory" "gopkg.in/segmentio/analytics-go.v3" diff --git a/components/ledger/pkg/analytics/backend.go b/components/ledger/pkg/analytics/backend.go index 50b92a1178..f682fc9a0a 100644 --- a/components/ledger/pkg/analytics/backend.go +++ b/components/ledger/pkg/analytics/backend.go @@ -3,8 +3,8 @@ package analytics import ( "context" - "github.com/formancehq/ledger/pkg/storage" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/google/uuid" "github.com/pkg/errors" @@ -38,7 +38,7 @@ type Backend interface { } type defaultBackend struct { - driver *storage.Driver + driver *driver.Driver appID string } @@ -75,7 +75,7 @@ func (d defaultBackend) GetLedgerStore(ctx context.Context, name string) (Ledger var _ Backend = (*defaultBackend)(nil) -func newDefaultBackend(driver *storage.Driver, appID string) *defaultBackend { +func newDefaultBackend(driver *driver.Driver, appID string) *defaultBackend { return &defaultBackend{ driver: driver, appID: appID, diff --git a/components/ledger/pkg/analytics/module.go b/components/ledger/pkg/analytics/module.go index 3ffe13389f..83314e909f 100644 --- a/components/ledger/pkg/analytics/module.go +++ b/components/ledger/pkg/analytics/module.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "go.uber.org/fx" "gopkg.in/segmentio/analytics-go.v3" ) @@ -18,7 +18,7 @@ func NewHeartbeatModule(version, writeKey, appID string, interval time.Duration) fx.Provide(func(client analytics.Client, backend Backend) *heartbeat { return newHeartbeat(backend, client, version, interval) }), - fx.Provide(func(driver *storage.Driver) Backend { + fx.Provide(func(driver *driver.Driver) Backend { return newDefaultBackend(driver, appID) }), fx.Invoke(func(m *heartbeat, lc fx.Lifecycle) { diff --git a/components/ledger/pkg/api/api.go b/components/ledger/pkg/api/api.go index 77a25f7e78..473badce25 100644 --- a/components/ledger/pkg/api/api.go +++ b/components/ledger/pkg/api/api.go @@ -8,7 +8,7 @@ import ( "github.com/formancehq/ledger/pkg/api/routes" "github.com/formancehq/ledger/pkg/ledger" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/stack/libs/go-libs/health" "go.opentelemetry.io/otel/metric" "go.uber.org/fx" @@ -21,7 +21,7 @@ type Config struct { func Module(cfg Config) fx.Option { return fx.Options( fx.Provide(routes.NewRouter), - fx.Provide(func(storageDriver *storage.Driver, resolver *ledger.Resolver) controllers.Backend { + fx.Provide(func(storageDriver *driver.Driver, resolver *ledger.Resolver) controllers.Backend { return controllers.NewDefaultBackend(storageDriver, cfg.Version, resolver) }), //TODO(gfyrag): Move in pkg/ledger package @@ -33,9 +33,9 @@ func Module(cfg Config) fx.Option { }) }), fx.Provide(fx.Annotate(metric.NewNoopMeterProvider, fx.As(new(metric.MeterProvider)))), - fx.Decorate(fx.Annotate(func(meterProvider metric.MeterProvider) (metrics.GlobalMetricsRegistry, error) { - return metrics.RegisterGlobalMetricsRegistry(meterProvider) - }, fx.As(new(metrics.GlobalMetricsRegistry)))), + fx.Decorate(fx.Annotate(func(meterProvider metric.MeterProvider) (metrics.GlobalRegistry, error) { + return metrics.RegisterGlobalRegistry(meterProvider) + }, fx.As(new(metrics.GlobalRegistry)))), health.Module(), ) } diff --git a/components/ledger/pkg/api/apierrors/errors.go b/components/ledger/pkg/api/apierrors/errors.go index e0ba73ae09..f8b6d67b08 100644 --- a/components/ledger/pkg/api/apierrors/errors.go +++ b/components/ledger/pkg/api/apierrors/errors.go @@ -10,7 +10,7 @@ import ( "github.com/formancehq/ledger/pkg/ledger/command" "github.com/formancehq/ledger/pkg/machine/vm" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/api" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/pkg/errors" diff --git a/components/ledger/pkg/api/controllers/account_controller.go b/components/ledger/pkg/api/controllers/account_controller.go index 546ab5d1b9..bebb4288a5 100644 --- a/components/ledger/pkg/api/controllers/account_controller.go +++ b/components/ledger/pkg/api/controllers/account_controller.go @@ -67,12 +67,6 @@ func GetAccounts(w http.ResponseWriter, r *http.Request) { } } - balanceOperator, err := getBalanceOperator(w, r) - if err != nil { - apierrors.ResponseError(w, r, err) - return - } - pageSize, err := getPageSize(r) if err != nil { apierrors.ResponseError(w, r, err) @@ -82,8 +76,6 @@ func GetAccounts(w http.ResponseWriter, r *http.Request) { accountsQuery = accountsQuery. WithAfterAddress(r.URL.Query().Get("after")). WithAddressFilter(r.URL.Query().Get("address")). - WithBalanceFilter(balance). - WithBalanceOperatorFilter(balanceOperator). WithMetadataFilter(sharedapi.GetQueryMap(r.URL.Query(), "metadata")). WithPageSize(pageSize) } diff --git a/components/ledger/pkg/api/controllers/account_controller_test.go b/components/ledger/pkg/api/controllers/account_controller_test.go index 6ef0969793..d6cc51df26 100644 --- a/components/ledger/pkg/api/controllers/account_controller_test.go +++ b/components/ledger/pkg/api/controllers/account_controller_test.go @@ -32,9 +32,8 @@ func TestGetAccounts(t *testing.T) { testCases := []testCase{ { - name: "nominal", - expectQuery: ledgerstore.NewAccountsQuery(). - WithBalanceOperatorFilter("gte"), + name: "nominal", + expectQuery: ledgerstore.NewAccountsQuery(), }, { name: "using metadata", @@ -42,7 +41,6 @@ func TestGetAccounts(t *testing.T) { "metadata[roles]": []string{"admin"}, }, expectQuery: ledgerstore.NewAccountsQuery(). - WithBalanceOperatorFilter("gte"). WithMetadataFilter(map[string]string{ "roles": "admin", }), @@ -53,7 +51,6 @@ func TestGetAccounts(t *testing.T) { "metadata[a.nested.key]": []string{"hello"}, }, expectQuery: ledgerstore.NewAccountsQuery(). - WithBalanceOperatorFilter("gte"). WithMetadataFilter(map[string]string{ "a.nested.key": "hello", }), @@ -64,55 +61,15 @@ func TestGetAccounts(t *testing.T) { "after": []string{"foo"}, }, expectQuery: ledgerstore.NewAccountsQuery(). - WithBalanceOperatorFilter("gte"). WithAfterAddress("foo"). WithMetadataFilter(map[string]string{}), }, - { - name: "using balance with default operator", - queryParams: url.Values{ - "balance": []string{"50"}, - }, - expectQuery: ledgerstore.NewAccountsQuery(). - WithBalanceOperatorFilter("gte"). - WithBalanceFilter("50"). - WithMetadataFilter(map[string]string{}), - }, - { - name: "using balance with specified operator", - queryParams: url.Values{ - "balance": []string{"50"}, - "balanceOperator": []string{"gt"}, - }, - expectQuery: ledgerstore.NewAccountsQuery(). - WithBalanceOperatorFilter("gt"). - WithBalanceFilter("50"). - WithMetadataFilter(map[string]string{}), - }, - { - name: "using invalid balance", - queryParams: url.Values{ - "balance": []string{"xxx"}, - }, - expectedErrorCode: apierrors.ErrValidation, - expectStatusCode: http.StatusBadRequest, - }, - { - name: "using balance with invalid operator", - queryParams: url.Values{ - "balance": []string{"50"}, - "balanceOperator": []string{"xxx"}, - }, - expectedErrorCode: apierrors.ErrValidation, - expectStatusCode: http.StatusBadRequest, - }, { name: "using address", queryParams: url.Values{ "address": []string{"foo"}, }, expectQuery: ledgerstore.NewAccountsQuery(). - WithBalanceOperatorFilter("gte"). WithAddressFilter("foo"). WithMetadataFilter(map[string]string{}), }, @@ -155,8 +112,7 @@ func TestGetAccounts(t *testing.T) { }, expectQuery: ledgerstore.NewAccountsQuery(). WithPageSize(controllers.MaxPageSize). - WithMetadataFilter(map[string]string{}). - WithBalanceOperatorFilter("gte"), + WithMetadataFilter(map[string]string{}), }, } for _, testCase := range testCases { @@ -183,7 +139,7 @@ func TestGetAccounts(t *testing.T) { Return(&expectedCursor, nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodGet, "/xxx/accounts", nil) rec := httptest.NewRecorder() @@ -193,11 +149,11 @@ func TestGetAccounts(t *testing.T) { require.Equal(t, testCase.expectStatusCode, rec.Code) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { - cursor := DecodeCursorResponse[core.Account](t, rec.Body) + cursor := sharedapi.DecodeCursorResponse[core.Account](t, rec.Body) require.Equal(t, expectedCursor, *cursor) } else { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) @@ -212,7 +168,7 @@ func TestGetAccount(t *testing.T) { Address: "foo", Metadata: metadata.Metadata{}, }, - Volumes: map[string]core.Volumes{}, + Volumes: core.VolumesByAssets{}, } backend, mock := newTestingBackend(t) @@ -220,7 +176,7 @@ func TestGetAccount(t *testing.T) { GetAccount(gomock.Any(), "foo"). Return(&account, nil) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodGet, "/xxx/accounts/foo", nil) rec := httptest.NewRecorder() @@ -228,7 +184,7 @@ func TestGetAccount(t *testing.T) { router.ServeHTTP(rec, req) require.Equal(t, http.StatusOK, rec.Code) - response, _ := DecodeSingleResponse[core.AccountWithVolumes](t, rec.Body) + response, _ := sharedapi.DecodeSingleResponse[core.AccountWithVolumes](t, rec.Body) require.Equal(t, account, response) } @@ -281,9 +237,9 @@ func TestPostAccountMetadata(t *testing.T) { Return(nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) - req := httptest.NewRequest(http.MethodPost, "/xxx/accounts/"+testCase.account+"/metadata", Buffer(t, testCase.body)) + req := httptest.NewRequest(http.MethodPost, "/xxx/accounts/"+testCase.account+"/metadata", sharedapi.Buffer(t, testCase.body)) rec := httptest.NewRecorder() req.URL.RawQuery = testCase.queryParams.Encode() @@ -292,7 +248,7 @@ func TestPostAccountMetadata(t *testing.T) { require.Equal(t, testCase.expectStatusCode, rec.Code) if testCase.expectStatusCode >= 300 || testCase.expectStatusCode < 200 { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) diff --git a/components/ledger/pkg/api/controllers/api.go b/components/ledger/pkg/api/controllers/api.go index 7604d2ed4f..c96f14a074 100644 --- a/components/ledger/pkg/api/controllers/api.go +++ b/components/ledger/pkg/api/controllers/api.go @@ -6,7 +6,7 @@ import ( "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/ledger" "github.com/formancehq/ledger/pkg/ledger/command" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/api" "github.com/formancehq/stack/libs/go-libs/metadata" @@ -18,11 +18,11 @@ type Ledger interface { GetAccount(ctx context.Context, param string) (*core.AccountWithVolumes, error) GetAccounts(ctx context.Context, query ledgerstore.AccountsQuery) (*api.Cursor[core.Account], error) CountAccounts(ctx context.Context, query ledgerstore.AccountsQuery) (uint64, error) - GetBalancesAggregated(ctx context.Context, q ledgerstore.BalancesQuery) (core.AssetsBalances, error) - GetBalances(ctx context.Context, q ledgerstore.BalancesQuery) (*api.Cursor[core.AccountsBalances], error) + GetBalancesAggregated(ctx context.Context, q ledgerstore.BalancesQuery) (core.BalancesByAssets, error) + GetBalances(ctx context.Context, q ledgerstore.BalancesQuery) (*api.Cursor[core.BalancesByAssetsByAccounts], error) GetMigrationsInfo(ctx context.Context) ([]core.MigrationInfo, error) Stats(ctx context.Context) (ledger.Stats, error) - GetLogs(ctx context.Context, query ledgerstore.LogsQuery) (*api.Cursor[core.PersistedLog], error) + GetLogs(ctx context.Context, query ledgerstore.LogsQuery) (*api.Cursor[core.ChainedLog], error) CountTransactions(ctx context.Context, query ledgerstore.TransactionsQuery) (uint64, error) GetTransactions(ctx context.Context, query ledgerstore.TransactionsQuery) (*api.Cursor[core.ExpandedTransaction], error) GetTransaction(ctx context.Context, id uint64) (*core.ExpandedTransaction, error) @@ -40,7 +40,7 @@ type Backend interface { } type DefaultBackend struct { - storageDriver *storage.Driver + storageDriver *driver.Driver resolver *ledger.Resolver version string } @@ -63,7 +63,7 @@ func (d DefaultBackend) GetVersion() string { var _ Backend = (*DefaultBackend)(nil) -func NewDefaultBackend(driver *storage.Driver, version string, resolver *ledger.Resolver) *DefaultBackend { +func NewDefaultBackend(driver *driver.Driver, version string, resolver *ledger.Resolver) *DefaultBackend { return &DefaultBackend{ storageDriver: driver, resolver: resolver, diff --git a/components/ledger/pkg/api/controllers/api_test.go b/components/ledger/pkg/api/controllers/api_test.go index d79f8e63aa..50460c2466 100644 --- a/components/ledger/pkg/api/controllers/api_test.go +++ b/components/ledger/pkg/api/controllers/api_test.go @@ -117,10 +117,10 @@ func (mr *MockLedgerMockRecorder) GetAccounts(ctx, query interface{}) *gomock.Ca } // GetBalances mocks base method. -func (m *MockLedger) GetBalances(ctx context.Context, q ledgerstore.BalancesQuery) (*api.Cursor[core.AccountsBalances], error) { +func (m *MockLedger) GetBalances(ctx context.Context, q ledgerstore.BalancesQuery) (*api.Cursor[core.BalancesByAssetsByAccounts], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetBalances", ctx, q) - ret0, _ := ret[0].(*api.Cursor[core.AccountsBalances]) + ret0, _ := ret[0].(*api.Cursor[core.BalancesByAssetsByAccounts]) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -132,10 +132,10 @@ func (mr *MockLedgerMockRecorder) GetBalances(ctx, q interface{}) *gomock.Call { } // GetBalancesAggregated mocks base method. -func (m *MockLedger) GetBalancesAggregated(ctx context.Context, q ledgerstore.BalancesQuery) (core.AssetsBalances, error) { +func (m *MockLedger) GetBalancesAggregated(ctx context.Context, q ledgerstore.BalancesQuery) (core.BalancesByAssets, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetBalancesAggregated", ctx, q) - ret0, _ := ret[0].(core.AssetsBalances) + ret0, _ := ret[0].(core.BalancesByAssets) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -147,10 +147,10 @@ func (mr *MockLedgerMockRecorder) GetBalancesAggregated(ctx, q interface{}) *gom } // GetLogs mocks base method. -func (m *MockLedger) GetLogs(ctx context.Context, query ledgerstore.LogsQuery) (*api.Cursor[core.PersistedLog], error) { +func (m *MockLedger) GetLogs(ctx context.Context, query ledgerstore.LogsQuery) (*api.Cursor[core.ChainedLog], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetLogs", ctx, query) - ret0, _ := ret[0].(*api.Cursor[core.PersistedLog]) + ret0, _ := ret[0].(*api.Cursor[core.ChainedLog]) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/components/ledger/pkg/api/controllers/balance_controller_test.go b/components/ledger/pkg/api/controllers/balance_controller_test.go index c31201b913..dc937d17f2 100644 --- a/components/ledger/pkg/api/controllers/balance_controller_test.go +++ b/components/ledger/pkg/api/controllers/balance_controller_test.go @@ -43,7 +43,7 @@ func TestGetBalancesAggregated(t *testing.T) { testCase := testCase t.Run(testCase.name, func(t *testing.T) { - expectedBalances := core.AssetsBalances{ + expectedBalances := core.BalancesByAssets{ "world": big.NewInt(-100), } backend, mock := newTestingBackend(t) @@ -51,7 +51,7 @@ func TestGetBalancesAggregated(t *testing.T) { GetBalancesAggregated(gomock.Any(), testCase.expectQuery). Return(expectedBalances, nil) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodGet, "/xxx/aggregate/balances", nil) rec := httptest.NewRecorder() @@ -60,7 +60,7 @@ func TestGetBalancesAggregated(t *testing.T) { router.ServeHTTP(rec, req) require.Equal(t, http.StatusOK, rec.Code) - balances, ok := DecodeSingleResponse[core.AssetsBalances](t, rec.Body) + balances, ok := sharedapi.DecodeSingleResponse[core.BalancesByAssets](t, rec.Body) require.True(t, ok) require.Equal(t, expectedBalances, balances) }) @@ -123,10 +123,10 @@ func TestGetBalances(t *testing.T) { testCase.expectStatusCode = http.StatusOK } - expectedCursor := sharedapi.Cursor[core.AccountsBalances]{ - Data: []core.AccountsBalances{ + expectedCursor := sharedapi.Cursor[core.BalancesByAssetsByAccounts]{ + Data: []core.BalancesByAssetsByAccounts{ { - "world": core.AssetsBalances{ + "world": core.BalancesByAssets{ "USD": big.NewInt(100), }, }, @@ -140,7 +140,7 @@ func TestGetBalances(t *testing.T) { Return(&expectedCursor, nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodGet, "/xxx/balances", nil) rec := httptest.NewRecorder() @@ -150,11 +150,11 @@ func TestGetBalances(t *testing.T) { require.Equal(t, testCase.expectStatusCode, rec.Code) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { - cursor := DecodeCursorResponse[core.AccountsBalances](t, rec.Body) + cursor := sharedapi.DecodeCursorResponse[core.BalancesByAssetsByAccounts](t, rec.Body) require.Equal(t, expectedCursor, *cursor) } else { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) diff --git a/components/ledger/pkg/api/controllers/config_controller_test.go b/components/ledger/pkg/api/controllers/config_controller_test.go index 492ee32f9e..22bf88da99 100644 --- a/components/ledger/pkg/api/controllers/config_controller_test.go +++ b/components/ledger/pkg/api/controllers/config_controller_test.go @@ -17,7 +17,7 @@ func TestGetInfo(t *testing.T) { t.Parallel() backend, _ := newTestingBackend(t) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) backend. EXPECT(). diff --git a/components/ledger/pkg/api/controllers/ledger_controller_test.go b/components/ledger/pkg/api/controllers/ledger_controller_test.go index 4ac3408365..c3ed8d540c 100644 --- a/components/ledger/pkg/api/controllers/ledger_controller_test.go +++ b/components/ledger/pkg/api/controllers/ledger_controller_test.go @@ -25,7 +25,7 @@ func TestGetLedgerInfo(t *testing.T) { t.Parallel() backend, mock := newTestingBackend(t) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) migrationInfo := []core.MigrationInfo{ { @@ -53,7 +53,7 @@ func TestGetLedgerInfo(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) - info, ok := DecodeSingleResponse[controllers.Info](t, rec.Body) + info, ok := sharedapi.DecodeSingleResponse[controllers.Info](t, rec.Body) require.True(t, ok) require.EqualValues(t, controllers.Info{ @@ -68,7 +68,7 @@ func TestGetStats(t *testing.T) { t.Parallel() backend, mock := newTestingBackend(t) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) expectedStats := ledger.Stats{ Transactions: 10, @@ -86,7 +86,7 @@ func TestGetStats(t *testing.T) { require.Equal(t, http.StatusOK, rec.Code) - stats, ok := DecodeSingleResponse[ledger.Stats](t, rec.Body) + stats, ok := sharedapi.DecodeSingleResponse[ledger.Stats](t, rec.Body) require.True(t, ok) require.EqualValues(t, expectedStats, stats) @@ -163,10 +163,10 @@ func TestGetLogs(t *testing.T) { testCase.expectStatusCode = http.StatusOK } - expectedCursor := sharedapi.Cursor[core.PersistedLog]{ - Data: []core.PersistedLog{ + expectedCursor := sharedapi.Cursor[core.ChainedLog]{ + Data: []core.ChainedLog{ *core.NewTransactionLog(core.NewTransaction(), map[string]metadata.Metadata{}). - ComputePersistentLog(nil), + ChainLog(nil), }, } @@ -177,7 +177,7 @@ func TestGetLogs(t *testing.T) { Return(&expectedCursor, nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodGet, "/xxx/logs", nil) rec := httptest.NewRecorder() @@ -187,7 +187,7 @@ func TestGetLogs(t *testing.T) { require.Equal(t, testCase.expectStatusCode, rec.Code) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { - cursor := DecodeCursorResponse[core.PersistedLog](t, rec.Body) + cursor := sharedapi.DecodeCursorResponse[core.ChainedLog](t, rec.Body) cursorData, err := json.Marshal(cursor) require.NoError(t, err) @@ -204,7 +204,7 @@ func TestGetLogs(t *testing.T) { require.Equal(t, expectedCursorAsMap, cursorAsMap) } else { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) diff --git a/components/ledger/pkg/api/controllers/query.go b/components/ledger/pkg/api/controllers/query.go index 1d1af6a3a4..69fe7568ef 100644 --- a/components/ledger/pkg/api/controllers/query.go +++ b/components/ledger/pkg/api/controllers/query.go @@ -12,7 +12,7 @@ import ( ) const ( - MaxPageSize = 1000 + MaxPageSize = 100 DefaultPageSize = ledgerstore.QueryDefaultPageSize QueryKeyCursor = "cursor" @@ -52,19 +52,6 @@ func getPageSize(r *http.Request) (uint64, error) { return pageSize, nil } -func getBalanceOperator(w http.ResponseWriter, r *http.Request) (ledgerstore.BalanceOperator, error) { - balanceOperator := ledgerstore.DefaultBalanceOperator - balanceOperatorStr := r.URL.Query().Get(QueryKeyBalanceOperator) - if balanceOperatorStr != "" { - var ok bool - if balanceOperator, ok = ledgerstore.NewBalanceOperator(balanceOperatorStr); !ok { - return "", errorsutil.NewError(command.ErrValidation, ErrInvalidBalanceOperator) - } - } - - return balanceOperator, nil -} - func getCommandParameters(r *http.Request) command.Parameters { dryRunAsString := r.URL.Query().Get("dryRun") dryRun := strings.ToUpper(dryRunAsString) == "YES" || strings.ToUpper(dryRunAsString) == "TRUE" || dryRunAsString == "1" diff --git a/components/ledger/pkg/api/controllers/transaction_controller_test.go b/components/ledger/pkg/api/controllers/transaction_controller_test.go index ec31318319..5903d9e304 100644 --- a/components/ledger/pkg/api/controllers/transaction_controller_test.go +++ b/components/ledger/pkg/api/controllers/transaction_controller_test.go @@ -232,9 +232,9 @@ func TestPostTransactions(t *testing.T) { Return(expectedTx, nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) - req := httptest.NewRequest(http.MethodPost, "/xxx/transactions", Buffer(t, testCase.payload)) + req := httptest.NewRequest(http.MethodPost, "/xxx/transactions", sharedapi.Buffer(t, testCase.payload)) rec := httptest.NewRecorder() req.URL.RawQuery = testCase.queryParams.Encode() @@ -242,12 +242,12 @@ func TestPostTransactions(t *testing.T) { require.Equal(t, testCase.expectedStatusCode, rec.Code) if testCase.expectedStatusCode < 300 && testCase.expectedStatusCode >= 200 { - tx, ok := DecodeSingleResponse[core.Transaction](t, rec.Body) + tx, ok := sharedapi.DecodeSingleResponse[core.Transaction](t, rec.Body) require.True(t, ok) require.Equal(t, *expectedTx, tx) } else { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) @@ -294,9 +294,9 @@ func TestPostTransactionMetadata(t *testing.T) { Return(nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) - req := httptest.NewRequest(http.MethodPost, "/xxx/transactions/0/metadata", Buffer(t, testCase.body)) + req := httptest.NewRequest(http.MethodPost, "/xxx/transactions/0/metadata", sharedapi.Buffer(t, testCase.body)) rec := httptest.NewRecorder() req.URL.RawQuery = testCase.queryParams.Encode() @@ -305,7 +305,7 @@ func TestPostTransactionMetadata(t *testing.T) { require.Equal(t, testCase.expectStatusCode, rec.Code) if testCase.expectStatusCode >= 300 || testCase.expectStatusCode < 200 { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) @@ -327,7 +327,7 @@ func TestGetTransaction(t *testing.T) { GetTransaction(gomock.Any(), uint64(0)). Return(&tx, nil) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodGet, "/xxx/transactions/0", nil) rec := httptest.NewRecorder() @@ -335,7 +335,7 @@ func TestGetTransaction(t *testing.T) { router.ServeHTTP(rec, req) require.Equal(t, http.StatusOK, rec.Code) - response, _ := DecodeSingleResponse[core.ExpandedTransaction](t, rec.Body) + response, _ := sharedapi.DecodeSingleResponse[core.ExpandedTransaction](t, rec.Body) require.Equal(t, tx, response) } @@ -524,7 +524,7 @@ func TestGetTransactions(t *testing.T) { Return(&expectedCursor, nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodGet, "/xxx/transactions", nil) rec := httptest.NewRecorder() @@ -534,11 +534,11 @@ func TestGetTransactions(t *testing.T) { require.Equal(t, testCase.expectStatusCode, rec.Code) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { - cursor := DecodeCursorResponse[core.ExpandedTransaction](t, rec.Body) + cursor := sharedapi.DecodeCursorResponse[core.ExpandedTransaction](t, rec.Body) require.Equal(t, expectedCursor, *cursor) } else { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) @@ -662,7 +662,7 @@ func TestCountTransactions(t *testing.T) { Return(uint64(10), nil) } - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodHead, "/xxx/transactions", nil) rec := httptest.NewRecorder() @@ -675,7 +675,7 @@ func TestCountTransactions(t *testing.T) { require.Equal(t, "10", rec.Header().Get("Count")) } else { err := sharedapi.ErrorResponse{} - Decode(t, rec.Body, &err) + sharedapi.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) } }) @@ -694,7 +694,7 @@ func TestRevertTransaction(t *testing.T) { RevertTransaction(gomock.Any(), command.Parameters{}, uint64(0)). Return(expectedTx, nil) - router := routes.NewRouter(backend, nil, nil, metrics.NewNoOpMetricsRegistry()) + router := routes.NewRouter(backend, nil, metrics.NewNoOpRegistry()) req := httptest.NewRequest(http.MethodPost, "/xxx/transactions/0/revert", nil) rec := httptest.NewRecorder() @@ -702,7 +702,7 @@ func TestRevertTransaction(t *testing.T) { router.ServeHTTP(rec, req) require.Equal(t, http.StatusCreated, rec.Code) - tx, ok := DecodeSingleResponse[core.Transaction](t, rec.Body) + tx, ok := sharedapi.DecodeSingleResponse[core.Transaction](t, rec.Body) require.True(t, ok) require.Equal(t, *expectedTx, tx) } diff --git a/components/ledger/pkg/api/controllers/utils_test.go b/components/ledger/pkg/api/controllers/utils_test.go index 342c9c907b..9fec5cf83b 100644 --- a/components/ledger/pkg/api/controllers/utils_test.go +++ b/components/ledger/pkg/api/controllers/utils_test.go @@ -1,44 +1,11 @@ package controllers_test import ( - "bytes" - "encoding/json" - "io" "testing" - sharedapi "github.com/formancehq/stack/libs/go-libs/api" "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) -func Encode(t *testing.T, v interface{}) []byte { - data, err := json.Marshal(v) - assert.NoError(t, err) - return data -} - -func Buffer(t *testing.T, v interface{}) *bytes.Buffer { - return bytes.NewBuffer(Encode(t, v)) -} - -func Decode(t *testing.T, reader io.Reader, v interface{}) { - err := json.NewDecoder(reader).Decode(v) - require.NoError(t, err) -} - -func DecodeSingleResponse[T any](t *testing.T, reader io.Reader) (T, bool) { - res := sharedapi.BaseResponse[T]{} - Decode(t, reader, &res) - return *res.Data, true -} - -func DecodeCursorResponse[T any](t *testing.T, reader io.Reader) *sharedapi.Cursor[T] { - res := sharedapi.BaseResponse[T]{} - Decode(t, reader, &res) - return res.Cursor -} - func newTestingBackend(t *testing.T) (*MockBackend, *MockLedger) { ctrl := gomock.NewController(t) mockLedger := NewMockLedger(ctrl) diff --git a/components/ledger/pkg/api/middlewares/ledger_middleware.go b/components/ledger/pkg/api/middlewares/ledger_middleware.go index b4c8a7b856..da6d75a944 100644 --- a/components/ledger/pkg/api/middlewares/ledger_middleware.go +++ b/components/ledger/pkg/api/middlewares/ledger_middleware.go @@ -1,17 +1,33 @@ package middlewares import ( + "math/rand" "net/http" + "time" "github.com/formancehq/ledger/pkg/api/apierrors" "github.com/formancehq/ledger/pkg/api/controllers" "github.com/formancehq/ledger/pkg/opentelemetry/tracer" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/go-chi/chi/v5" - "github.com/google/uuid" - "go.opentelemetry.io/otel/trace" ) +var r *rand.Rand + +func init() { + r = rand.New(rand.NewSource(time.Now().UnixNano())) +} + +var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") + +func randomTraceID(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letterRunes[r.Intn(len(letterRunes))] + } + return string(b) +} + func LedgerMiddleware( resolver controllers.Backend, ) func(handler http.Handler) http.Handler { @@ -27,7 +43,16 @@ func LedgerMiddleware( defer span.End() r = r.WithContext(ctx) - r = wrapRequest(r) + + loggerFields := map[string]any{ + "ledger": name, + } + if span.SpanContext().TraceID().IsValid() { + loggerFields["trace-id"] = span.SpanContext().TraceID().String() + } else { + loggerFields["trace-id"] = randomTraceID(10) + } + r = r.WithContext(logging.ContextWithFields(r.Context(), loggerFields)) l, err := resolver.GetLedger(r.Context(), name) if err != nil { @@ -45,14 +70,3 @@ func LedgerMiddleware( }) } } - -func wrapRequest(r *http.Request) *http.Request { - span := trace.SpanFromContext(r.Context()) - contextKeyID := uuid.NewString() - if span.SpanContext().SpanID().IsValid() { - contextKeyID = span.SpanContext().SpanID().String() - } - return r.WithContext(logging.ContextWithLogger(r.Context(), logging.FromContext(r.Context()).WithFields(map[string]any{ - "contextID": contextKeyID, - }))) -} diff --git a/components/ledger/pkg/api/middlewares/metrics_middleware.go b/components/ledger/pkg/api/middlewares/metrics_middleware.go index 5a97f2ff56..f6360cfd10 100644 --- a/components/ledger/pkg/api/middlewares/metrics_middleware.go +++ b/components/ledger/pkg/api/middlewares/metrics_middleware.go @@ -24,7 +24,7 @@ func (r *statusRecorder) WriteHeader(status int) { r.ResponseWriter.WriteHeader(status) } -func MetricsMiddleware(globalMetricsRegistry metrics.GlobalMetricsRegistry) func(h http.Handler) http.Handler { +func MetricsMiddleware(globalMetricsRegistry metrics.GlobalRegistry) func(h http.Handler) http.Handler { return func(h http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { attrs := []attribute.KeyValue{} diff --git a/components/ledger/pkg/api/routes/routes.go b/components/ledger/pkg/api/routes/routes.go index 8cf3e370d1..197a1559d1 100644 --- a/components/ledger/pkg/api/routes/routes.go +++ b/components/ledger/pkg/api/routes/routes.go @@ -7,7 +7,6 @@ import ( "github.com/formancehq/ledger/pkg/api/middlewares" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" "github.com/formancehq/stack/libs/go-libs/health" - "github.com/formancehq/stack/libs/go-libs/logging" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/cors" @@ -16,9 +15,8 @@ import ( func NewRouter( backend controllers.Backend, - logger logging.Logger, healthController *health.HealthController, - globalMetricsRegistry metrics.GlobalMetricsRegistry, + globalMetricsRegistry metrics.GlobalRegistry, ) chi.Router { router := chi.NewMux() @@ -29,16 +27,6 @@ func NewRouter( }, AllowCredentials: true, }).Handler, - func(handler http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if logger != nil { - r = r.WithContext( - logging.ContextWithLogger(r.Context(), logger), - ) - } - handler.ServeHTTP(w, r) - }) - }, middlewares.MetricsMiddleware(globalMetricsRegistry), middleware.Recoverer, ) diff --git a/components/ledger/pkg/bus/aggregate.go b/components/ledger/pkg/bus/aggregate.go deleted file mode 100644 index def651d604..0000000000 --- a/components/ledger/pkg/bus/aggregate.go +++ /dev/null @@ -1,23 +0,0 @@ -package bus - -import ( - "github.com/formancehq/ledger/pkg/core" -) - -func aggregatePostCommitVolumes(txs ...core.ExpandedTransaction) core.AccountsAssetsVolumes { - ret := core.AccountsAssetsVolumes{} - for i := len(txs) - 1; i >= 0; i-- { - tx := txs[i] - for _, posting := range tx.Postings { - if !ret.HasAccountAndAsset(posting.Source, posting.Asset) { - ret.SetVolumes(posting.Source, posting.Asset, - tx.PostCommitVolumes.GetVolumes(posting.Source, posting.Asset)) - } - if !ret.HasAccountAndAsset(posting.Destination, posting.Asset) { - ret.SetVolumes(posting.Destination, posting.Asset, - tx.PostCommitVolumes.GetVolumes(posting.Destination, posting.Asset)) - } - } - } - return ret -} diff --git a/components/ledger/pkg/bus/message.go b/components/ledger/pkg/bus/message.go index 6e0863e823..469f6d9138 100644 --- a/components/ledger/pkg/bus/message.go +++ b/components/ledger/pkg/bus/message.go @@ -23,12 +23,8 @@ type EventMessage struct { } type CommittedTransactions struct { - Ledger string `json:"ledger"` - Transactions []core.ExpandedTransaction `json:"transactions"` - // Deprecated (use postCommitVolumes) - Volumes core.AccountsAssetsVolumes `json:"volumes"` - PostCommitVolumes core.AccountsAssetsVolumes `json:"postCommitVolumes"` - PreCommitVolumes core.AccountsAssetsVolumes `json:"preCommitVolumes"` + Ledger string `json:"ledger"` + Transactions []core.Transaction `json:"transactions"` } func newEventCommittedTransactions(txs CommittedTransactions) EventMessage { @@ -59,9 +55,9 @@ func newEventSavedMetadata(metadata SavedMetadata) EventMessage { } type RevertedTransaction struct { - Ledger string `json:"ledger"` - RevertedTransaction core.ExpandedTransaction `json:"revertedTransaction"` - RevertTransaction core.ExpandedTransaction `json:"revertTransaction"` + Ledger string `json:"ledger"` + RevertedTransaction core.Transaction `json:"revertedTransaction"` + RevertTransaction core.Transaction `json:"revertTransaction"` } func newEventRevertedTransaction(tx RevertedTransaction) EventMessage { diff --git a/components/ledger/pkg/bus/module.go b/components/ledger/pkg/bus/module.go deleted file mode 100644 index 378194b0c7..0000000000 --- a/components/ledger/pkg/bus/module.go +++ /dev/null @@ -1,10 +0,0 @@ -package bus - -import ( - "github.com/formancehq/ledger/pkg/ledger/monitor" - "go.uber.org/fx" -) - -func LedgerMonitorModule() fx.Option { - return fx.Decorate(fx.Annotate(newLedgerMonitor, fx.As(new(monitor.Monitor)))) -} diff --git a/components/ledger/pkg/bus/monitor.go b/components/ledger/pkg/bus/monitor.go index 5dbf62867e..19a76c32dd 100644 --- a/components/ledger/pkg/bus/monitor.go +++ b/components/ledger/pkg/bus/monitor.go @@ -5,7 +5,7 @@ import ( "github.com/ThreeDotsLabs/watermill/message" "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/ledger/monitor" + "github.com/formancehq/ledger/pkg/ledger/query" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/formancehq/stack/libs/go-libs/publish" @@ -13,43 +13,41 @@ import ( type ledgerMonitor struct { publisher message.Publisher + ledgerName string } -var _ monitor.Monitor = &ledgerMonitor{} +var _ query.Monitor = &ledgerMonitor{} -func newLedgerMonitor(publisher message.Publisher) *ledgerMonitor { +func NewLedgerMonitor(publisher message.Publisher, ledgerName string) *ledgerMonitor { m := &ledgerMonitor{ publisher: publisher, + ledgerName: ledgerName, } return m } -func (l *ledgerMonitor) CommittedTransactions(ctx context.Context, ledger string, txs ...core.ExpandedTransaction) { - postCommitVolumes := aggregatePostCommitVolumes(txs...) +func (l *ledgerMonitor) CommittedTransactions(ctx context.Context, txs ...core.Transaction) { l.publish(ctx, EventTypeCommittedTransactions, newEventCommittedTransactions(CommittedTransactions{ - Ledger: ledger, - Transactions: txs, - Volumes: postCommitVolumes, - PostCommitVolumes: postCommitVolumes, - PreCommitVolumes: core.AggregatePreCommitVolumes(txs...), + Ledger: l.ledgerName, + Transactions: txs, })) } -func (l *ledgerMonitor) SavedMetadata(ctx context.Context, ledger, targetType, targetID string, metadata metadata.Metadata) { +func (l *ledgerMonitor) SavedMetadata(ctx context.Context, targetType, targetID string, metadata metadata.Metadata) { l.publish(ctx, EventTypeSavedMetadata, newEventSavedMetadata(SavedMetadata{ - Ledger: ledger, + Ledger: l.ledgerName, TargetType: targetType, TargetID: targetID, Metadata: metadata, })) } -func (l *ledgerMonitor) RevertedTransaction(ctx context.Context, ledger string, reverted, revert *core.ExpandedTransaction) { +func (l *ledgerMonitor) RevertedTransaction(ctx context.Context, reverted, revert *core.Transaction) { l.publish(ctx, EventTypeRevertedTransaction, newEventRevertedTransaction(RevertedTransaction{ - Ledger: ledger, + Ledger: l.ledgerName, RevertedTransaction: *reverted, RevertTransaction: *revert, })) diff --git a/components/ledger/pkg/bus/monitor_test.go b/components/ledger/pkg/bus/monitor_test.go index f35b3f57eb..9072759563 100644 --- a/components/ledger/pkg/bus/monitor_test.go +++ b/components/ledger/pkg/bus/monitor_test.go @@ -25,8 +25,8 @@ func TestMonitor(t *testing.T) { p := publish.NewTopicMapperPublisherDecorator(pubSub, map[string]string{ "*": "testing", }) - m := newLedgerMonitor(p) - go m.CommittedTransactions(context.Background(), uuid.New()) + m := NewLedgerMonitor(p, uuid.New()) + go m.CommittedTransactions(context.Background()) select { case m := <-messages: diff --git a/components/ledger/pkg/core/account.go b/components/ledger/pkg/core/account.go index 82a6a7d4ba..c5abb30f65 100644 --- a/components/ledger/pkg/core/account.go +++ b/components/ledger/pkg/core/account.go @@ -30,7 +30,7 @@ func NewAccount(address string) Account { type AccountWithVolumes struct { Account - Volumes AssetsVolumes `json:"volumes"` + Volumes VolumesByAssets `json:"volumes"` } func NewAccountWithVolumes(address string) *AccountWithVolumes { @@ -39,7 +39,7 @@ func NewAccountWithVolumes(address string) *AccountWithVolumes { Address: address, Metadata: metadata.Metadata{}, }, - Volumes: map[string]Volumes{}, + Volumes: map[string]*Volumes{}, } } @@ -47,7 +47,7 @@ func (v AccountWithVolumes) MarshalJSON() ([]byte, error) { type aux AccountWithVolumes return json.Marshal(struct { aux - Balances AssetsBalances `json:"balances"` + Balances BalancesByAssets `json:"balances"` }{ aux: aux(v), Balances: v.Volumes.Balances(), diff --git a/components/ledger/pkg/core/log.go b/components/ledger/pkg/core/log.go index b86d6d8462..5ecb4998ff 100644 --- a/components/ledger/pkg/core/log.go +++ b/components/ledger/pkg/core/log.go @@ -80,19 +80,20 @@ type hashable interface { hashString(buf *buffer) } -type PersistedLog struct { +type ChainedLog struct { Log - ID uint64 `json:"id"` - Hash []byte `json:"hash"` + ID uint64 `json:"id"` + Projected bool `json:"-"` + Hash []byte `json:"hash"` } -func (l *PersistedLog) WithID(id uint64) *PersistedLog { +func (l *ChainedLog) WithID(id uint64) *ChainedLog { l.ID = id return l } -func (l *PersistedLog) UnmarshalJSON(data []byte) error { - type auxLog PersistedLog +func (l *ChainedLog) UnmarshalJSON(data []byte) error { + type auxLog ChainedLog type log struct { auxLog Data json.RawMessage `json:"data"` @@ -107,18 +108,18 @@ func (l *PersistedLog) UnmarshalJSON(data []byte) error { if err != nil { return err } - *l = PersistedLog(rawLog.auxLog) + *l = ChainedLog(rawLog.auxLog) return err } -func (l *PersistedLog) ComputeHash(previous *PersistedLog) { +func (l *ChainedLog) ComputeHash(previous *ChainedLog) { buf := bufferPool.Get().(*buffer) defer func() { buf.reset() bufferPool.Put(buf) }() - hashLog := func(l *PersistedLog) { + hashLog := func(l *ChainedLog) { buf.writeUInt64(l.ID) buf.writeUInt16(uint16(l.Type)) buf.writeUInt64(uint64(l.Date.UnixNano())) @@ -157,8 +158,8 @@ func (l *Log) WithIdempotencyKey(key string) *Log { return l } -func (l *Log) ComputePersistentLog(previous *PersistedLog) *PersistedLog { - ret := &PersistedLog{} +func (l *Log) ChainLog(previous *ChainedLog) *ChainedLog { + ret := &ChainedLog{} ret.Log = *l ret.ComputeHash(previous) if previous != nil { @@ -315,18 +316,19 @@ func HydrateLog(_type LogType, data []byte) (hashable, error) { type Accounts map[string]Account type ActiveLog struct { - *Log - Ingested chan struct{} + *ChainedLog + Projected chan struct{} `json:"-"` } -func (h *ActiveLog) SetIngested() { - close(h.Ingested) +func (h *ActiveLog) SetProjected() { + h.ChainedLog.Projected = true + close(h.Projected) } -func NewActiveLog(log *Log) *ActiveLog { +func NewActiveLog(log *ChainedLog) *ActiveLog { return &ActiveLog{ - Log: log, - Ingested: make(chan struct{}), + ChainedLog: log, + Projected: make(chan struct{}), } } @@ -389,17 +391,15 @@ var ( ) type LogPersistenceTracker struct { - activeLog *ActiveLog - done chan struct{} - persistedLog *PersistedLog + activeLog *ActiveLog + done chan struct{} } func (r *LogPersistenceTracker) ActiveLog() *ActiveLog { return r.activeLog } -func (r *LogPersistenceTracker) Resolve(persistedLog *PersistedLog) { - r.persistedLog = persistedLog +func (r *LogPersistenceTracker) Resolve() { close(r.done) } @@ -407,10 +407,6 @@ func (r *LogPersistenceTracker) Done() chan struct{} { return r.done } -func (r *LogPersistenceTracker) Result() *PersistedLog { - return r.persistedLog -} - func NewLogPersistenceTracker(log *ActiveLog) *LogPersistenceTracker { return &LogPersistenceTracker{ activeLog: log, @@ -418,8 +414,8 @@ func NewLogPersistenceTracker(log *ActiveLog) *LogPersistenceTracker { } } -func NewResolvedLogPersistenceTracker(log *ActiveLog, v *PersistedLog) *LogPersistenceTracker { +func NewResolvedLogPersistenceTracker(log *ActiveLog) *LogPersistenceTracker { ret := NewLogPersistenceTracker(log) - ret.Resolve(v) + ret.Resolve() return ret } diff --git a/components/ledger/pkg/core/move.go b/components/ledger/pkg/core/move.go new file mode 100644 index 0000000000..a30fc8eb85 --- /dev/null +++ b/components/ledger/pkg/core/move.go @@ -0,0 +1,15 @@ +package core + +import ( + "math/big" +) + +type Move struct { + TransactionID uint64 + Amount *big.Int + Asset string + Account string + PostingIndex uint8 + IsSource bool + Timestamp Time +} diff --git a/components/ledger/pkg/core/time.go b/components/ledger/pkg/core/time.go index 1558d58524..41abca40e4 100644 --- a/components/ledger/pkg/core/time.go +++ b/components/ledger/pkg/core/time.go @@ -75,6 +75,10 @@ func (t Time) Round(precision time.Duration) Time { } } +func (t Time) Equal(t2 Time) bool { + return t.Time.Equal(t2.Time) +} + func (t Time) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf(`"%s"`, t.Format(DateFormat))), nil } diff --git a/components/ledger/pkg/core/transaction.go b/components/ledger/pkg/core/transaction.go index 0981c47971..d6f8d17133 100644 --- a/components/ledger/pkg/core/transaction.go +++ b/components/ledger/pkg/core/transaction.go @@ -80,6 +80,29 @@ func (t *Transaction) WithMetadata(m metadata.Metadata) *Transaction { return t } +func (t Transaction) GetMoves() []*Move { + ret := make([]*Move, 0) + for ind, posting := range t.Postings { + ret = append(ret, &Move{ + TransactionID: t.ID, + Amount: posting.Amount, + Asset: posting.Asset, + Account: posting.Source, + PostingIndex: uint8(ind), + IsSource: true, + Timestamp: t.Timestamp, + }, &Move{ + TransactionID: t.ID, + Amount: posting.Amount, + Asset: posting.Asset, + Account: posting.Destination, + PostingIndex: uint8(ind), + Timestamp: t.Timestamp, + }) + } + return ret +} + func (t *Transaction) hashString(buf *buffer) { buf.writeUInt64(t.ID) t.TransactionData.hashString(buf) diff --git a/components/ledger/pkg/core/transaction_test.go b/components/ledger/pkg/core/transaction_test.go index f0ec17c0d8..3bd0390aaf 100644 --- a/components/ledger/pkg/core/transaction_test.go +++ b/components/ledger/pkg/core/transaction_test.go @@ -137,12 +137,12 @@ func TestReverseTransaction(t *testing.T) { } func BenchmarkHash(b *testing.B) { - logs := make([]PersistedLog, b.N) - var previous *PersistedLog + logs := make([]ChainedLog, b.N) + var previous *ChainedLog for i := 0; i < b.N; i++ { newLog := NewTransactionLog(NewTransaction().WithPostings( NewPosting("world", "bank", "USD", big.NewInt(100)), - ), map[string]metadata.Metadata{}).ComputePersistentLog(previous) + ), map[string]metadata.Metadata{}).ChainLog(previous) previous = newLog logs = append(logs, *newLog) } diff --git a/components/ledger/pkg/core/volumes.go b/components/ledger/pkg/core/volumes.go index 4f91bb9fd7..277294a7c5 100644 --- a/components/ledger/pkg/core/volumes.go +++ b/components/ledger/pkg/core/volumes.go @@ -11,7 +11,7 @@ type Volumes struct { Output *big.Int `json:"output"` } -func (v Volumes) CopyWithZerosIfNeeded() Volumes { +func (v Volumes) CopyWithZerosIfNeeded() *Volumes { var input *big.Int if v.Input == nil { input = &big.Int{} @@ -24,29 +24,46 @@ func (v Volumes) CopyWithZerosIfNeeded() Volumes { } else { output = new(big.Int).Set(v.Output) } - return Volumes{ + return &Volumes{ Input: input, Output: output, } } -func (v Volumes) WithInput(input *big.Int) Volumes { +func (v Volumes) WithInput(input *big.Int) *Volumes { v.Input = input - return v + return &v } -func (v Volumes) WithOutput(output *big.Int) Volumes { +func (v Volumes) WithInputInt64(value int64) *Volumes { + v.Input = big.NewInt(value) + return &v +} + +func (v Volumes) WithOutput(output *big.Int) *Volumes { v.Output = output - return v + return &v +} + +func (v Volumes) WithOutputInt64(value int64) *Volumes { + v.Output = big.NewInt(value) + return &v } -func NewEmptyVolumes() Volumes { - return Volumes{ +func NewEmptyVolumes() *Volumes { + return &Volumes{ Input: new(big.Int), Output: new(big.Int), } } +func NewVolumesInt64(input, output int64) *Volumes { + return &Volumes{ + Input: big.NewInt(input), + Output: big.NewInt(output), + } +} + type VolumesWithBalance struct { Input *big.Int `json:"input"` Output *big.Int `json:"output"` @@ -73,63 +90,63 @@ func (v Volumes) Balance() *big.Int { return new(big.Int).Sub(input, output) } -func (v Volumes) copy() Volumes { - return Volumes{ +func (v Volumes) copy() *Volumes { + return &Volumes{ Input: new(big.Int).Set(v.Input), Output: new(big.Int).Set(v.Output), } } -type AssetsBalances map[string]*big.Int +type BalancesByAssets map[string]*big.Int -type AssetsVolumes map[string]Volumes +type VolumesByAssets map[string]*Volumes -type AccountsBalances map[string]AssetsBalances +type BalancesByAssetsByAccounts map[string]BalancesByAssets -func (v AssetsVolumes) Balances() AssetsBalances { - balances := AssetsBalances{} +func (v VolumesByAssets) Balances() BalancesByAssets { + balances := BalancesByAssets{} for asset, vv := range v { balances[asset] = new(big.Int).Sub(vv.Input, vv.Output) } return balances } -func (v AssetsVolumes) copy() AssetsVolumes { - ret := AssetsVolumes{} +func (v VolumesByAssets) copy() VolumesByAssets { + ret := VolumesByAssets{} for key, volumes := range v { ret[key] = volumes.copy() } return ret } -type AccountsAssetsVolumes map[string]AssetsVolumes +type AccountsAssetsVolumes map[string]VolumesByAssets -func (a AccountsAssetsVolumes) GetVolumes(account, asset string) Volumes { +func (a AccountsAssetsVolumes) GetVolumes(account, asset string) *Volumes { if a == nil { - return Volumes{ + return &Volumes{ Input: &big.Int{}, Output: &big.Int{}, } } if assetsVolumes, ok := a[account]; !ok { - return Volumes{ + return &Volumes{ Input: &big.Int{}, Output: &big.Int{}, } } else { - return Volumes{ + return &Volumes{ Input: assetsVolumes[asset].Input, Output: assetsVolumes[asset].Output, } } } -func (a *AccountsAssetsVolumes) SetVolumes(account, asset string, volumes Volumes) { +func (a *AccountsAssetsVolumes) SetVolumes(account, asset string, volumes *Volumes) { if *a == nil { *a = AccountsAssetsVolumes{} } if assetsVolumes, ok := (*a)[account]; !ok { - (*a)[account] = map[string]Volumes{ + (*a)[account] = map[string]*Volumes{ asset: volumes.CopyWithZerosIfNeeded(), } } else { @@ -142,7 +159,7 @@ func (a *AccountsAssetsVolumes) AddInput(account, asset string, input *big.Int) *a = AccountsAssetsVolumes{} } if assetsVolumes, ok := (*a)[account]; !ok { - (*a)[account] = map[string]Volumes{ + (*a)[account] = map[string]*Volumes{ asset: { Input: input, Output: &big.Int{}, @@ -160,7 +177,7 @@ func (a *AccountsAssetsVolumes) AddOutput(account, asset string, output *big.Int *a = AccountsAssetsVolumes{} } if assetsVolumes, ok := (*a)[account]; !ok { - (*a)[account] = map[string]Volumes{ + (*a)[account] = map[string]*Volumes{ asset: { Output: output, Input: &big.Int{}, diff --git a/components/ledger/pkg/ledger/aggregator/aggregator.go b/components/ledger/pkg/ledger/aggregator/aggregator.go deleted file mode 100644 index 5742931836..0000000000 --- a/components/ledger/pkg/ledger/aggregator/aggregator.go +++ /dev/null @@ -1,98 +0,0 @@ -package aggregator - -import ( - "context" - "math/big" - - "github.com/formancehq/ledger/pkg/core" - "github.com/pkg/errors" -) - -type TxVolumeAggregator struct { - agg *VolumeAggregator - previousTx *TxVolumeAggregator - - PreCommitVolumes core.AccountsAssetsVolumes - PostCommitVolumes core.AccountsAssetsVolumes -} - -func (tva *TxVolumeAggregator) FindInPreviousTxs(addr, asset string) *core.Volumes { - current := tva.previousTx - for current != nil { - if v, ok := current.PostCommitVolumes[addr][asset]; ok { - return &v - } - current = current.previousTx - } - return nil -} - -func (tva *TxVolumeAggregator) Transfer( - ctx context.Context, - from, to, asset string, - amount *big.Int, -) error { - for _, addr := range []string{from, to} { - if !tva.PreCommitVolumes.HasAccountAndAsset(addr, asset) { - previousVolumes := tva.FindInPreviousTxs(addr, asset) - - if previousVolumes != nil { - tva.PreCommitVolumes.SetVolumes(addr, asset, *previousVolumes) - } else { - acc, err := tva.agg.store.GetAccountWithVolumes(ctx, addr) - if err != nil { - return errors.Wrap(err, "getting account while transferring") - } - tva.PreCommitVolumes.SetVolumes(addr, asset, acc.Volumes[asset]) - } - } - if !tva.PostCommitVolumes.HasAccountAndAsset(addr, asset) { - tva.PostCommitVolumes.SetVolumes(addr, asset, tva.PreCommitVolumes.GetVolumes(addr, asset)) - } - } - tva.PostCommitVolumes.AddOutput(from, asset, amount) - tva.PostCommitVolumes.AddInput(to, asset, amount) - - return nil -} - -func (agg *TxVolumeAggregator) AddPostings(ctx context.Context, postings ...core.Posting) error { - for _, posting := range postings { - if err := agg.Transfer(ctx, posting.Source, posting.Destination, posting.Asset, posting.Amount); err != nil { - return errors.Wrap(err, "aggregating volumes") - } - } - return nil -} - -type VolumeAggregator struct { - txs []*TxVolumeAggregator - store Store -} - -func (agg *VolumeAggregator) NextTx() *TxVolumeAggregator { - var previousTx *TxVolumeAggregator - if len(agg.txs) > 0 { - previousTx = agg.txs[len(agg.txs)-1] - } - tva := &TxVolumeAggregator{ - agg: agg, - previousTx: previousTx, - } - agg.txs = append(agg.txs, tva) - return tva -} - -func (agg *VolumeAggregator) NextTxWithPostings(ctx context.Context, postings ...core.Posting) (*TxVolumeAggregator, error) { - tva := agg.NextTx() - if err := tva.AddPostings(ctx, postings...); err != nil { - return nil, err - } - return tva, nil -} - -func Volumes(store Store) *VolumeAggregator { - return &VolumeAggregator{ - store: store, - } -} diff --git a/components/ledger/pkg/ledger/aggregator/store.go b/components/ledger/pkg/ledger/aggregator/store.go deleted file mode 100644 index 38d187f824..0000000000 --- a/components/ledger/pkg/ledger/aggregator/store.go +++ /dev/null @@ -1,11 +0,0 @@ -package aggregator - -import ( - "context" - - "github.com/formancehq/ledger/pkg/core" -) - -type Store interface { - GetAccountWithVolumes(ctx context.Context, address string) (*core.AccountWithVolumes, error) -} diff --git a/components/ledger/pkg/ledger/command/commander.go b/components/ledger/pkg/ledger/command/commander.go index f0ff5066b1..3a4d3b5fc5 100644 --- a/components/ledger/pkg/ledger/command/commander.go +++ b/components/ledger/pkg/ledger/command/commander.go @@ -10,7 +10,7 @@ import ( "github.com/formancehq/ledger/pkg/machine" "github.com/formancehq/ledger/pkg/machine/vm" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/collectionutils" "github.com/formancehq/stack/libs/go-libs/errorsutil" "github.com/formancehq/stack/libs/go-libs/metadata" @@ -26,11 +26,14 @@ type Parameters struct { type Commander struct { store Store locker Locker - metricsRegistry metrics.PerLedgerMetricsRegistry + metricsRegistry metrics.PerLedgerRegistry compiler *Compiler running sync.WaitGroup lastTXID *atomic.Int64 referencer *Referencer + mu sync.Mutex + + lastLog *core.ChainedLog } func New( @@ -38,12 +41,13 @@ func New( locker Locker, compiler *Compiler, referencer *Referencer, - metricsRegistry metrics.PerLedgerMetricsRegistry, + metricsRegistry metrics.PerLedgerRegistry, ) *Commander { log, err := store.ReadLastLogWithType(context.Background(), core.NewTransactionLogType, core.RevertedTransactionLogType) if err != nil && !storageerrors.IsNotFoundError(err) { panic(err) } + var lastTxID *uint64 if err == nil { switch payload := log.Data.(type) { @@ -61,6 +65,12 @@ func New( } else { lastTXID.Add(-1) } + + lastLog, err := store.GetLastLog(context.Background()) + if err != nil && !storageerrors.IsNotFoundError(err) { + panic(err) + } + return &Commander{ store: store, locker: locker, @@ -68,6 +78,7 @@ func New( compiler: compiler, referencer: referencer, lastTXID: lastTXID, + lastLog: lastLog, } } @@ -76,7 +87,7 @@ func (commander *Commander) GetLedgerStore() Store { } func (commander *Commander) exec(ctx context.Context, parameters Parameters, script core.RunScript, - logComputer func(tx *core.Transaction, accountMetadata map[string]metadata.Metadata) *core.Log) (*core.PersistedLog, error) { + logComputer func(tx *core.Transaction, accountMetadata map[string]metadata.Metadata) *core.Log) (*core.ChainedLog, error) { if script.Script.Plain == "" { return nil, ErrNoScript @@ -160,12 +171,12 @@ func (commander *Commander) exec(ctx context.Context, parameters Parameters, scr log = log.WithIdempotencyKey(parameters.IdempotencyKey) } - return executionContext.AppendLog(ctx, core.NewActiveLog(log)) + return executionContext.AppendLog(ctx, log) }) if err != nil { return nil, err } - return tracker.Result(), nil + return tracker.ActiveLog().ChainedLog, nil } func (commander *Commander) CreateTransaction(ctx context.Context, parameters Parameters, script core.RunScript) (*core.Transaction, error) { @@ -215,7 +226,7 @@ func (commander *Commander) SaveMeta(ctx context.Context, parameters Parameters, return nil, errorsutil.NewError(ErrValidation, errors.Errorf("unknown target type '%s'", targetType)) } - return executionContext.AppendLog(ctx, core.NewActiveLog(log)) + return executionContext.AppendLog(ctx, log) }) return err } @@ -266,3 +277,11 @@ func (commander *Commander) RevertTransaction(ctx context.Context, parameters Pa func (commander *Commander) Wait() { commander.running.Wait() } + +func (commander *Commander) chainLog(log *core.Log) *core.ChainedLog { + commander.mu.Lock() + defer commander.mu.Unlock() + + commander.lastLog = log.ChainLog(commander.lastLog) + return commander.lastLog +} diff --git a/components/ledger/pkg/ledger/command/commander_test.go b/components/ledger/pkg/ledger/command/commander_test.go index 13d5b0de8c..85a025e747 100644 --- a/components/ledger/pkg/ledger/command/commander_test.go +++ b/components/ledger/pkg/ledger/command/commander_test.go @@ -6,7 +6,7 @@ import ( "testing" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/collectionutils" "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/pkg/errors" @@ -14,7 +14,14 @@ import ( ) type mockStore struct { - logs []*core.PersistedLog + logs []*core.ChainedLog +} + +func (m *mockStore) GetLastLog(ctx context.Context) (*core.ChainedLog, error) { + if len(m.logs) == 0 { + return nil, nil + } + return m.logs[len(m.logs)-1], nil } func (m *mockStore) GetBalanceFromLogs(ctx context.Context, address, asset string) (*big.Int, error) { @@ -63,8 +70,8 @@ func (m *mockStore) GetMetadataFromLogs(ctx context.Context, address, key string return "", errors.New("not found") } -func (m *mockStore) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*core.PersistedLog, error) { - first := collectionutils.First(m.logs, func(log *core.PersistedLog) bool { +func (m *mockStore) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*core.ChainedLog, error) { + first := collectionutils.First(m.logs, func(log *core.ChainedLog) bool { return log.IdempotencyKey == key }) if first == nil { @@ -73,8 +80,8 @@ func (m *mockStore) ReadLogWithIdempotencyKey(ctx context.Context, key string) ( return first, nil } -func (m *mockStore) ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.PersistedLog, error) { - first := collectionutils.First(m.logs, func(log *core.PersistedLog) bool { +func (m *mockStore) ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.ChainedLog, error) { + first := collectionutils.First(m.logs, func(log *core.ChainedLog) bool { if log.Type != core.NewTransactionLogType { return false } @@ -86,8 +93,8 @@ func (m *mockStore) ReadLogForCreatedTransactionWithReference(ctx context.Contex return first, nil } -func (m *mockStore) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) { - first := collectionutils.First(m.logs, func(log *core.PersistedLog) bool { +func (m *mockStore) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.ChainedLog, error) { + first := collectionutils.First(m.logs, func(log *core.ChainedLog) bool { if log.Type != core.NewTransactionLogType { return false } @@ -99,8 +106,8 @@ func (m *mockStore) ReadLogForCreatedTransaction(ctx context.Context, txID uint6 return first, nil } -func (m *mockStore) ReadLogForRevertedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) { - first := collectionutils.First(m.logs, func(log *core.PersistedLog) bool { +func (m *mockStore) ReadLogForRevertedTransaction(ctx context.Context, txID uint64) (*core.ChainedLog, error) { + first := collectionutils.First(m.logs, func(log *core.ChainedLog) bool { if log.Type != core.RevertedTransactionLogType { return false } @@ -112,8 +119,8 @@ func (m *mockStore) ReadLogForRevertedTransaction(ctx context.Context, txID uint return first, nil } -func (m *mockStore) ReadLastLogWithType(background context.Context, logType ...core.LogType) (*core.PersistedLog, error) { - first := collectionutils.First(m.logs, func(log *core.PersistedLog) bool { +func (m *mockStore) ReadLastLogWithType(background context.Context, logType ...core.LogType) (*core.ChainedLog, error) { + first := collectionutils.First(m.logs, func(log *core.ChainedLog) bool { return collectionutils.Contains(logType, log.Type) }) if first == nil { @@ -124,17 +131,17 @@ func (m *mockStore) ReadLastLogWithType(background context.Context, logType ...c func (m *mockStore) AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) { var ( - previous, persistedLog *core.PersistedLog + previous, persistedLog *core.ChainedLog ) if len(m.logs) > 0 { previous = m.logs[len(m.logs)-1] } - persistedLog = log.ComputePersistentLog(previous) + persistedLog = log.ChainLog(previous) m.logs = append(m.logs, persistedLog) ret := core.NewLogPersistenceTracker(log) - ret.Resolve(persistedLog) - log.SetIngested() + ret.Resolve() + log.SetProjected() return ret, nil } @@ -146,7 +153,7 @@ var ( func newMockStore() *mockStore { return &mockStore{ - logs: []*core.PersistedLog{}, + logs: []*core.ChainedLog{}, } } @@ -197,7 +204,7 @@ var testCases = []testCase{ WithPostings(core.NewPosting("world", "mint", "GEM", big.NewInt(100))). WithReference("tx_ref") log := core.NewTransactionLog(tx, nil) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log)) + _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) }, script: ` @@ -262,7 +269,7 @@ var testCases = []testCase{ WithTimestamp(now), map[string]metadata.Metadata{}, ).WithIdempotencyKey("testing") - _, err := r.AppendLog(context.Background(), core.NewActiveLog(log)) + _, err := r.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) }, parameters: Parameters{ @@ -325,7 +332,7 @@ func TestRevert(t *testing.T) { ), map[string]metadata.Metadata{}, ) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log)) + _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) ledger := New(store, NoOpLocker, NewCompiler(1024), NewReferencer(), nil) @@ -338,7 +345,7 @@ func TestRevertWithAlreadyReverted(t *testing.T) { store := newMockStore() log := core. NewRevertedTransactionLog(core.Now(), 0, core.NewTransaction()) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log)) + _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) ledger := New(store, NoOpLocker, NewCompiler(1024), NewReferencer(), nil) @@ -356,7 +363,7 @@ func TestRevertWithRevertOccurring(t *testing.T) { ), map[string]metadata.Metadata{}, ) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log)) + _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) referencer := NewReferencer() diff --git a/components/ledger/pkg/ledger/command/context.go b/components/ledger/pkg/ledger/command/context.go index 7e9a792564..1ec3c5d1ba 100644 --- a/components/ledger/pkg/ledger/command/context.go +++ b/components/ledger/pkg/ledger/command/context.go @@ -4,7 +4,8 @@ import ( "context" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/stack/libs/go-libs/logging" ) type executionContext struct { @@ -12,11 +13,17 @@ type executionContext struct { parameters Parameters } -func (e *executionContext) AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) { +func (e *executionContext) AppendLog(ctx context.Context, log *core.Log) (*core.LogPersistenceTracker, error) { if e.parameters.DryRun { - return core.NewResolvedLogPersistenceTracker(log, log.ComputePersistentLog(nil)), nil + chainedLog := log.ChainLog(nil) + return core.NewResolvedLogPersistenceTracker(core.NewActiveLog(chainedLog)), nil } - return e.commander.store.AppendLog(ctx, log) + + activeLog := core.NewActiveLog(e.commander.chainLog(log)) + logging.FromContext(ctx).WithFields(map[string]any{ + "id": activeLog.ChainedLog.ID, + }).Debugf("Appending log") + return e.commander.store.AppendLog(ctx, activeLog) } func (e *executionContext) run(ctx context.Context, executor func(e *executionContext) (*core.LogPersistenceTracker, error)) (*core.LogPersistenceTracker, error) { @@ -26,9 +33,9 @@ func (e *executionContext) run(ctx context.Context, executor func(e *executionCo } defer e.commander.referencer.release(referenceIks, ik) - persistedLog, err := e.commander.store.ReadLogWithIdempotencyKey(ctx, ik) + chainedLog, err := e.commander.store.ReadLogWithIdempotencyKey(ctx, ik) if err == nil { - return core.NewResolvedLogPersistenceTracker(nil, persistedLog), nil + return core.NewResolvedLogPersistenceTracker(core.NewActiveLog(chainedLog)), nil } if err != storageerrors.ErrNotFound && err != nil { return nil, err @@ -39,8 +46,13 @@ func (e *executionContext) run(ctx context.Context, executor func(e *executionCo return nil, err } <-tracker.Done() + logger := logging.FromContext(ctx).WithFields(map[string]any{ + "id": tracker.ActiveLog().ChainedLog.ID, + }) + logger.Debugf("Log inserted in database") if !e.parameters.Async { - <-tracker.ActiveLog().Ingested + <-tracker.ActiveLog().Projected + logger.Debugf("Log fully ingested") } return tracker, nil } diff --git a/components/ledger/pkg/ledger/command/lock.go b/components/ledger/pkg/ledger/command/lock.go index 1d44db8f82..cd3f36b8ec 100644 --- a/components/ledger/pkg/ledger/command/lock.go +++ b/components/ledger/pkg/ledger/command/lock.go @@ -4,6 +4,10 @@ import ( "context" "sync" "sync/atomic" + + "github.com/formancehq/stack/libs/go-libs/collectionutils" + "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/pkg/errors" ) type Unlock func(ctx context.Context) @@ -26,47 +30,12 @@ type Accounts struct { Write []string } -type linkedListItem[T any] struct { - next *linkedListItem[T] - previous *linkedListItem[T] - value T - list *linkedList[T] -} - -func (i *linkedListItem[T]) Remove() { - if i.previous == nil { - i.list.first = i.next - return - } - if i.next == nil { - i.list.last = i.previous - return - } - i.previous.next, i.next.previous = i.next, i.previous -} - -type linkedList[T any] struct { - first *linkedListItem[T] - last *linkedListItem[T] -} - -func (l *linkedList[T]) PutTail(v T) *linkedListItem[T] { - item := &linkedListItem[T]{ - previous: l.last, - value: v, - list: l, - } - l.last = item - - return item -} - type lockIntent struct { accounts Accounts acquired chan struct{} } -func (intent *lockIntent) tryLock(chain *DefaultLocker) bool { +func (intent *lockIntent) tryLock(ctx context.Context, chain *DefaultLocker) bool { for _, account := range intent.accounts.Read { _, ok := chain.writeLocks[account] @@ -86,6 +55,8 @@ func (intent *lockIntent) tryLock(chain *DefaultLocker) bool { } } + logging.FromContext(ctx).Debugf("Lock acquired, read: %s, write: %s", intent.accounts.Read, intent.accounts.Write) + for _, account := range intent.accounts.Read { atomicValue, ok := chain.readLocks[account] if !ok { @@ -101,7 +72,8 @@ func (intent *lockIntent) tryLock(chain *DefaultLocker) bool { return true } -func (intent *lockIntent) unlock(chain *DefaultLocker) { +func (intent *lockIntent) unlock(ctx context.Context, chain *DefaultLocker) { + logging.FromContext(ctx).Debugf("Unlock accounts, read: %s, write: %s", intent.accounts.Read, intent.accounts.Write) for _, account := range intent.accounts.Read { atomicValue := chain.readLocks[account] if atomicValue.Add(-1) == 0 { @@ -114,7 +86,7 @@ func (intent *lockIntent) unlock(chain *DefaultLocker) { } type DefaultLocker struct { - intents *linkedList[*lockIntent] + intents *collectionutils.LinkedList[*lockIntent] mu sync.Mutex readLocks map[string]*atomic.Int64 writeLocks map[string]struct{} @@ -122,40 +94,73 @@ type DefaultLocker struct { func (defaultLocker *DefaultLocker) Lock(ctx context.Context, accounts Accounts) (Unlock, error) { defaultLocker.mu.Lock() + defer defaultLocker.mu.Unlock() + + logger := logging.FromContext(ctx).WithFields(map[string]any{ + "read": accounts.Read, + "write": accounts.Write, + }) + + logger.Debugf("Intent lock") intent := &lockIntent{ accounts: accounts, acquired: make(chan struct{}), } - item := defaultLocker.intents.PutTail(intent) - acquired := intent.tryLock(defaultLocker) - defaultLocker.mu.Unlock() + if acquired := intent.tryLock(logging.ContextWithLogger(ctx, logger), defaultLocker); !acquired { + logger.Debugf("Lock not acquired, some accounts are already used") - if !acquired { + defaultLocker.intents.Append(intent) select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, errors.Wrapf(ctx.Err(), "locking accounts: %s as read, and %s as write", accounts.Read, accounts.Write) case <-intent.acquired: + return func(ctx context.Context) { + defaultLocker.mu.Lock() + defer defaultLocker.mu.Unlock() + + intent.unlock(ctx, defaultLocker) + node := defaultLocker.intents.RemoveValue(intent) + + if node == nil { + panic("node should not be nil") + } + + for { + node = node.Next() + if node == nil { + break + } + if node.Value().tryLock(ctx, defaultLocker) { + close(node.Value().acquired) + } + } + }, nil } - } - - return func(ctx context.Context) { - defaultLocker.mu.Lock() - defer defaultLocker.mu.Unlock() - - intent.unlock(defaultLocker) - item.Remove() - - for next := item.next; next != nil; { - if next.value.tryLock(defaultLocker) { - close(next.value.acquired) + } else { + logger.Debugf("Lock directly acquired") + return func(ctx context.Context) { + defaultLocker.mu.Lock() + defer defaultLocker.mu.Unlock() + + intent.unlock(ctx, defaultLocker) + + node := defaultLocker.intents.FirstNode() + for { + if node == nil { + break + } + if node.Value().tryLock(ctx, defaultLocker) { + close(node.Value().acquired) + } + node = node.Next() } - } - }, nil + }, nil + } } func NewDefaultLocker() *DefaultLocker { return &DefaultLocker{ - intents: &linkedList[*lockIntent]{}, + intents: collectionutils.NewLinkedList[*lockIntent](), readLocks: map[string]*atomic.Int64{}, writeLocks: map[string]struct{}{}, } diff --git a/components/ledger/pkg/ledger/command/store.go b/components/ledger/pkg/ledger/command/store.go index 666287ac83..c08f3a7b43 100644 --- a/components/ledger/pkg/ledger/command/store.go +++ b/components/ledger/pkg/ledger/command/store.go @@ -2,192 +2,18 @@ package command import ( "context" - "math/big" - "sync" - "sync/atomic" "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/machine/vm" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" ) type Store interface { vm.Store AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) - ReadLastLogWithType(ctx context.Context, logType ...core.LogType) (*core.PersistedLog, error) - ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.PersistedLog, error) - ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) - ReadLogForRevertedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) - ReadLogWithIdempotencyKey(ctx context.Context, key string) (*core.PersistedLog, error) + ReadLastLogWithType(ctx context.Context, logType ...core.LogType) (*core.ChainedLog, error) + ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.ChainedLog, error) + ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.ChainedLog, error) + ReadLogForRevertedTransaction(ctx context.Context, txID uint64) (*core.ChainedLog, error) + ReadLogWithIdempotencyKey(ctx context.Context, key string) (*core.ChainedLog, error) + GetLastLog(ctx context.Context) (*core.ChainedLog, error) } - -type alwaysEmptyStore struct{} - -func (e alwaysEmptyStore) GetBalanceFromLogs(ctx context.Context, address, asset string) (*big.Int, error) { - return new(big.Int), nil -} - -func (e alwaysEmptyStore) GetMetadataFromLogs(ctx context.Context, address, key string) (string, error) { - return "", storageerrors.ErrNotFound -} - -func (e alwaysEmptyStore) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*core.PersistedLog, error) { - return nil, storageerrors.ErrNotFound -} - -func (e alwaysEmptyStore) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) { - return nil, storageerrors.ErrNotFound -} - -func (e alwaysEmptyStore) ReadLogForRevertedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) { - return nil, storageerrors.ErrNotFound -} - -func (e alwaysEmptyStore) AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) { - return core.NewResolvedLogPersistenceTracker(log, log.ComputePersistentLog(nil)), nil -} - -func (e alwaysEmptyStore) ReadLastLogWithType(ctx context.Context, logType ...core.LogType) (*core.PersistedLog, error) { - return nil, storageerrors.ErrNotFound -} - -func (e alwaysEmptyStore) ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.PersistedLog, error) { - return nil, storageerrors.ErrNotFound -} - -var _ Store = (*alwaysEmptyStore)(nil) - -var AlwaysEmptyStore = &alwaysEmptyStore{} - -type cacheEntry[T any] struct { - sync.Mutex - value T - inUse atomic.Int64 - ready chan struct{} -} - -func getEntry[T any](i *inMemoryStore, key string, valuer func() (T, error)) *cacheEntry[T] { - v, loaded := i.entries.LoadOrStore(key, &cacheEntry[T]{ - ready: make(chan struct{}), - }) - entry := v.(*cacheEntry[T]) - if !loaded { - var err error - entry.value, err = valuer() - if err != nil { - panic(err) - } - close(entry.ready) - - return entry - } - <-entry.ready - return entry -} - -type inMemoryStore struct { - Store - entries sync.Map -} - -func (i *inMemoryStore) getBalanceEntry(ctx context.Context, address, asset string) *cacheEntry[*big.Int] { - return getEntry(i, "accounts/"+address+"/"+asset, func() (*big.Int, error) { - return i.Store.GetBalanceFromLogs(ctx, address, asset) - }) -} - -func (i *inMemoryStore) getMetadataEntry(ctx context.Context, address, key string) *cacheEntry[string] { - return getEntry(i, "metadata/"+address+"/"+key, func() (string, error) { - return i.Store.GetMetadataFromLogs(ctx, address, key) - }) -} - -func (i *inMemoryStore) deleteBalanceEntry(address, asset string) { - i.entries.Delete("accounts/" + address + "/" + asset) -} - -func (i *inMemoryStore) deleteMetadataEntry(address, key string) { - i.entries.Delete("metadata/" + address + "/" + key) -} - -func (i *inMemoryStore) AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) { - switch payload := log.Data.(type) { - case core.NewTransactionLogPayload: - for _, posting := range payload.Transaction.Postings { - entry := i.getBalanceEntry(ctx, posting.Source, posting.Asset) - entry.Lock() - entry.inUse.Add(1) - entry.value.Add(entry.value, new(big.Int).Neg(posting.Amount)) - entry.Unlock() - - entry = i.getBalanceEntry(ctx, posting.Destination, posting.Asset) - entry.Lock() - entry.inUse.Add(1) - entry.value.Add(entry.value, posting.Amount) - entry.Unlock() - } - for address, metadata := range payload.AccountMetadata { - for key, value := range metadata { - entry := i.getMetadataEntry(ctx, address, key) - entry.Lock() - entry.inUse.Add(1) - entry.value = value - entry.Unlock() - } - } - case core.SetMetadataLogPayload: - if payload.TargetType == core.MetaTargetTypeAccount { - for key, value := range payload.Metadata { - entry := i.getMetadataEntry(ctx, payload.TargetID.(string), key) - entry.Lock() - entry.inUse.Add(1) - entry.value = value - entry.Unlock() - } - } - } - tracker, err := i.Store.AppendLog(ctx, log) - if err != nil { - return nil, err - } - go func() { - <-tracker.Done() - switch payload := log.Data.(type) { - case core.NewTransactionLogPayload: - for _, posting := range payload.Transaction.Postings { - if i.getBalanceEntry(ctx, posting.Source, posting.Asset).inUse.Add(-1) == 0 { - i.deleteBalanceEntry(posting.Source, posting.Asset) - } - if i.getBalanceEntry(ctx, posting.Destination, posting.Asset).inUse.Add(-1) == 0 { - i.deleteBalanceEntry(posting.Destination, posting.Asset) - } - } - for address, metadata := range payload.AccountMetadata { - for key := range metadata { - if i.getMetadataEntry(ctx, address, key).inUse.Add(-1) == 0 { - i.deleteMetadataEntry(address, key) - } - } - } - case core.SetMetadataLogPayload: - if payload.TargetType == core.MetaTargetTypeAccount { - for key := range payload.Metadata { - if i.getMetadataEntry(ctx, payload.TargetID.(string), key).inUse.Add(-1) == 0 { - i.deleteMetadataEntry(payload.TargetID.(string), key) - } - } - } - } - }() - return tracker, nil -} - -func (i *inMemoryStore) GetBalanceFromLogs(ctx context.Context, address, asset string) (*big.Int, error) { - return i.getBalanceEntry(ctx, address, asset).value, nil -} - -func (i *inMemoryStore) GetMetadataFromLogs(ctx context.Context, address, key string) (string, error) { - return i.getMetadataEntry(ctx, address, key).value, nil -} - -var _ Store = (*inMemoryStore)(nil) diff --git a/components/ledger/pkg/ledger/ledger.go b/components/ledger/pkg/ledger/ledger.go index 8c18fced02..fd661930b2 100644 --- a/components/ledger/pkg/ledger/ledger.go +++ b/components/ledger/pkg/ledger/ledger.go @@ -9,48 +9,50 @@ import ( "github.com/formancehq/ledger/pkg/opentelemetry/metrics" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/api" + "github.com/formancehq/stack/libs/go-libs/logging" "github.com/pkg/errors" ) type Ledger struct { *command.Commander - store *ledgerstore.Store - queryWorker *query.Worker - locker *command.DefaultLocker + store *ledgerstore.Store + projector *query.Projector + locker *command.DefaultLocker } func New( store *ledgerstore.Store, locker *command.DefaultLocker, - queryWorker *query.Worker, + queryWorker *query.Projector, compiler *command.Compiler, - metricsRegistry metrics.PerLedgerMetricsRegistry, + metricsRegistry metrics.PerLedgerRegistry, ) *Ledger { - store.OnLogWrote(func(logs []*ledgerstore.AppendedLog) { - if err := queryWorker.QueueLog(context.Background(), logs...); err != nil { + store.OnLogWrote(func(logs []*core.ActiveLog) { + if err := queryWorker.QueueLog(logs...); err != nil { panic(err) } }) return &Ledger{ - Commander: command.New(store, locker, compiler, command.NewReferencer(), metricsRegistry), - store: store, - queryWorker: queryWorker, - locker: locker, + Commander: command.New(store, locker, compiler, command.NewReferencer(), metricsRegistry), + store: store, + projector: queryWorker, + locker: locker, } } func (l *Ledger) Close(ctx context.Context) error { + logging.FromContext(ctx).Debugf("Close commander") l.Commander.Wait() - if err := l.queryWorker.Stop(ctx); err != nil { - return errors.Wrap(err, "stopping query worker") - } - - if err := l.store.Stop(ctx); err != nil { + logging.FromContext(ctx).Debugf("Close storage worker") + if err := l.store.Stop(logging.ContextWithField(ctx, "component", "store")); err != nil { return errors.Wrap(err, "stopping ledger store") } + logging.FromContext(ctx).Debugf("Close projector") + l.projector.Stop(logging.ContextWithField(ctx, "component", "projector")) + return nil } @@ -84,17 +86,17 @@ func (l *Ledger) GetAccount(ctx context.Context, address string) (*core.AccountW return accounts, errors.Wrap(err, "getting account") } -func (l *Ledger) GetBalances(ctx context.Context, q ledgerstore.BalancesQuery) (*api.Cursor[core.AccountsBalances], error) { +func (l *Ledger) GetBalances(ctx context.Context, q ledgerstore.BalancesQuery) (*api.Cursor[core.BalancesByAssetsByAccounts], error) { balances, err := l.store.GetBalances(ctx, q) return balances, errors.Wrap(err, "getting balances") } -func (l *Ledger) GetBalancesAggregated(ctx context.Context, q ledgerstore.BalancesQuery) (core.AssetsBalances, error) { +func (l *Ledger) GetBalancesAggregated(ctx context.Context, q ledgerstore.BalancesQuery) (core.BalancesByAssets, error) { balances, err := l.store.GetBalancesAggregated(ctx, q) return balances, errors.Wrap(err, "getting balances aggregated") } -func (l *Ledger) GetLogs(ctx context.Context, q ledgerstore.LogsQuery) (*api.Cursor[core.PersistedLog], error) { +func (l *Ledger) GetLogs(ctx context.Context, q ledgerstore.LogsQuery) (*api.Cursor[core.ChainedLog], error) { logs, err := l.store.GetLogs(ctx, q) return logs, errors.Wrap(err, "getting logs") } diff --git a/components/ledger/pkg/ledger/module.go b/components/ledger/pkg/ledger/module.go index 24d61420ea..9f6ef1e900 100644 --- a/components/ledger/pkg/ledger/module.go +++ b/components/ledger/pkg/ledger/module.go @@ -1,11 +1,12 @@ package ledger import ( + "github.com/ThreeDotsLabs/watermill/message" "github.com/formancehq/ledger/pkg/ledger/command" - "github.com/formancehq/ledger/pkg/ledger/monitor" "github.com/formancehq/ledger/pkg/ledger/query" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" + "github.com/formancehq/stack/libs/go-libs/logging" "go.uber.org/fx" ) @@ -13,36 +14,29 @@ type NumscriptCacheConfiguration struct { MaxCount int } -type QueryConfiguration struct { - LimitReadLogs int -} - type Configuration struct { NumscriptCache NumscriptCacheConfiguration - Query QueryConfiguration } func Module(configuration Configuration) fx.Option { return fx.Options( fx.Provide(func( - storageDriver *storage.Driver, - monitor monitor.Monitor, - metricsRegistry metrics.GlobalMetricsRegistry, + storageDriver *driver.Driver, + publisher message.Publisher, + metricsRegistry metrics.GlobalRegistry, + logger logging.Logger, ) *Resolver { options := []option{ - WithMonitor(monitor), + WithMessagePublisher(publisher), WithMetricsRegistry(metricsRegistry), + WithLogger(logger), } if configuration.NumscriptCache.MaxCount != 0 { options = append(options, WithCompiler(command.NewCompiler(configuration.NumscriptCache.MaxCount))) } return NewResolver(storageDriver, options...) }), - fx.Provide(fx.Annotate(monitor.NewNoOpMonitor, fx.As(new(monitor.Monitor)))), - fx.Provide(fx.Annotate(metrics.NewNoOpMetricsRegistry, fx.As(new(metrics.GlobalMetricsRegistry)))), - query.InitModule(), - fx.Decorate(func() *query.InitLedgerConfig { - return query.NewInitLedgerConfig(configuration.Query.LimitReadLogs) - }), + fx.Provide(fx.Annotate(query.NewNoOpMonitor, fx.As(new(query.Monitor)))), + fx.Provide(fx.Annotate(metrics.NewNoOpRegistry, fx.As(new(metrics.GlobalRegistry)))), ) } diff --git a/components/ledger/pkg/ledger/monitor/monitor.go b/components/ledger/pkg/ledger/monitor/monitor.go deleted file mode 100644 index 34ec249283..0000000000 --- a/components/ledger/pkg/ledger/monitor/monitor.go +++ /dev/null @@ -1,29 +0,0 @@ -package monitor - -import ( - "context" - - "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/stack/libs/go-libs/metadata" -) - -type Monitor interface { - CommittedTransactions(ctx context.Context, ledger string, res ...core.ExpandedTransaction) - SavedMetadata(ctx context.Context, ledger, targetType, id string, metadata metadata.Metadata) - RevertedTransaction(ctx context.Context, ledger string, reverted, revert *core.ExpandedTransaction) -} - -type noOpMonitor struct{} - -func (n noOpMonitor) CommittedTransactions(ctx context.Context, s string, res ...core.ExpandedTransaction) { -} -func (n noOpMonitor) SavedMetadata(ctx context.Context, ledger string, targetType string, id string, metadata metadata.Metadata) { -} -func (n noOpMonitor) RevertedTransaction(ctx context.Context, ledger string, reverted, revert *core.ExpandedTransaction) { -} - -var _ Monitor = &noOpMonitor{} - -func NewNoOpMonitor() *noOpMonitor { - return &noOpMonitor{} -} diff --git a/components/ledger/pkg/ledger/query/errors.go b/components/ledger/pkg/ledger/query/errors.go deleted file mode 100644 index 6d3dbb4af9..0000000000 --- a/components/ledger/pkg/ledger/query/errors.go +++ /dev/null @@ -1,11 +0,0 @@ -package query - -import "github.com/pkg/errors" - -var ( - ErrStorage = errors.New("storage error") -) - -func IsStorageError(err error) bool { - return errors.Is(err, ErrStorage) -} diff --git a/components/ledger/pkg/ledger/query/init.go b/components/ledger/pkg/ledger/query/init.go deleted file mode 100644 index 33b13640c2..0000000000 --- a/components/ledger/pkg/ledger/query/init.go +++ /dev/null @@ -1,143 +0,0 @@ -package query - -import ( - "context" - - "github.com/formancehq/ledger/pkg/ledger/monitor" - "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" - "github.com/formancehq/stack/libs/go-libs/errorsutil" - "github.com/pkg/errors" - "golang.org/x/sync/errgroup" -) - -type InitLedger struct { - cfg *InitLedgerConfig - driver *storage.Driver - monitor monitor.Monitor - metricsRegistry metrics.PerLedgerMetricsRegistry -} - -type InitLedgerConfig struct { - LimitReadLogs int -} - -func NewInitLedgerConfig(limitReadLogs int) *InitLedgerConfig { - return &InitLedgerConfig{ - LimitReadLogs: limitReadLogs, - } -} - -func NewDefaultInitLedgerConfig() *InitLedgerConfig { - return &InitLedgerConfig{ - LimitReadLogs: 10000, - } -} - -func initLedgers( - ctx context.Context, - cfg *InitLedgerConfig, - driver *storage.Driver, - monitor monitor.Monitor, - metricsRegistry metrics.PerLedgerMetricsRegistry, -) error { - ledgers, err := driver.GetSystemStore().ListLedgers(ctx) - if err != nil { - return err - } - - eg, ctxGroup := errgroup.WithContext(ctx) - for _, ledger := range ledgers { - _ledger := ledger - eg.Go(func() error { - store, err := driver.GetLedgerStore(ctxGroup, _ledger) - if err != nil && !storageerrors.IsNotFoundError(err) { - return err - } - - if storageerrors.IsNotFoundError(err) { - return nil - } - - if !store.IsInitialized() { - return nil - } - - if _, err := initLedger( - ctxGroup, - cfg, - _ledger, - NewDefaultStore(store), - monitor, - metricsRegistry, - ); err != nil { - return err - } - - return nil - }) - } - - return eg.Wait() -} - -func initLedger( - ctx context.Context, - cfg *InitLedgerConfig, - ledgerName string, - store Store, - monitor monitor.Monitor, - metricsRegistry metrics.PerLedgerMetricsRegistry, -) (uint64, error) { - if !store.IsInitialized() { - return 0, nil - } - - lastReadLogID, err := store.GetNextLogID(ctx) - if err != nil && !storageerrors.IsNotFoundError(err) { - return 0, errorsutil.NewError(ErrStorage, - errors.Wrap(err, "reading last log")) - } - - lastProcessedLogID := uint64(0) - for { - logs, err := store.ReadLogsRange(ctx, lastReadLogID, lastReadLogID+uint64(cfg.LimitReadLogs)) - if err != nil { - return 0, errorsutil.NewError(ErrStorage, - errors.Wrap(err, "reading logs since last ID")) - } - - if len(logs) == 0 { - // No logs, nothing to do - return lastProcessedLogID, nil - } - - if err := processLogs(ctx, ledgerName, store, monitor, logs...); err != nil { - return 0, errors.Wrap(err, "processing logs") - } - - metricsRegistry.QueryProcessedLogs().Add(ctx, int64(len(logs))) - - if err := store.UpdateNextLogID(ctx, logs[len(logs)-1].ID+1); err != nil { - return 0, errorsutil.NewError(ErrStorage, - errors.Wrap(err, "updating last read log")) - } - lastReadLogID = logs[len(logs)-1].ID + 1 - lastProcessedLogID = logs[len(logs)-1].ID - - if len(logs) < cfg.LimitReadLogs { - // Nothing to do anymore, no need to read more logs - return lastProcessedLogID, nil - } - - } -} - -func NewInitLedgers(cfg *InitLedgerConfig, driver *storage.Driver, monitor monitor.Monitor) *InitLedger { - return &InitLedger{ - cfg: cfg, - driver: driver, - monitor: monitor, - } -} diff --git a/components/ledger/pkg/ledger/query/module.go b/components/ledger/pkg/ledger/query/module.go deleted file mode 100644 index 401ac5c7e0..0000000000 --- a/components/ledger/pkg/ledger/query/module.go +++ /dev/null @@ -1,27 +0,0 @@ -package query - -import ( - "context" - - "go.uber.org/fx" -) - -func InitModule() fx.Option { - return fx.Options( - fx.Provide(NewDefaultInitLedgerConfig), - fx.Provide(NewInitLedgers), - fx.Invoke(func(lc fx.Lifecycle, initQuery *InitLedger) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - return initLedgers( - ctx, - initQuery.cfg, - initQuery.driver, - initQuery.monitor, - initQuery.metricsRegistry, - ) - }, - }) - }), - ) -} diff --git a/components/ledger/pkg/ledger/query/module_test.go b/components/ledger/pkg/ledger/query/module_test.go deleted file mode 100644 index be08c1ebbd..0000000000 --- a/components/ledger/pkg/ledger/query/module_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package query - -import ( - "context" - "math/big" - "testing" - - "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/ledger/monitor" - "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/stack/libs/go-libs/metadata" - "github.com/stretchr/testify/require" -) - -func TestInitQuery(t *testing.T) { - t.Parallel() - - now := core.Now() - - tx0 := core.NewTransaction().WithPostings( - core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), - ) - tx1 := core.NewTransaction().WithPostings( - core.NewPosting("bank", "user:1", "USD/2", big.NewInt(10)), - ) - - appliedMetadataOnTX1 := metadata.Metadata{ - "paymentID": "1234", - } - appliedMetadataOnAccount := metadata.Metadata{ - "category": "gold", - } - - log0 := core.NewTransactionLog(tx0, nil) - log1 := core.NewTransactionLog(tx1, nil) - log2 := core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeTransaction, - TargetID: tx1.ID, - Metadata: appliedMetadataOnTX1, - }) - log3 := core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeAccount, - TargetID: "bank", - Metadata: appliedMetadataOnAccount, - }) - log4 := core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeAccount, - TargetID: "another:account", - Metadata: appliedMetadataOnAccount, - }) - - logs := make([]*core.PersistedLog, 0) - var previous *core.PersistedLog - for _, l := range []*core.Log{ - log0, log1, log2, log3, log4, - } { - next := l.ComputePersistentLog(previous) - logs = append(logs, next) - previous = next - } - - ledgerStore := &mockStore{ - accounts: map[string]*core.AccountWithVolumes{}, - logs: logs, - } - - nextLogID, err := ledgerStore.GetNextLogID(context.Background()) - require.NoError(t, err) - require.Equal(t, uint64(0), nextLogID) - - lastProcessedId, err := initLedger( - context.Background(), - &InitLedgerConfig{ - LimitReadLogs: 2, - }, - "default_test", - ledgerStore, - monitor.NewNoOpMonitor(), - metrics.NewNoOpMetricsRegistry(), - ) - require.NoError(t, err) - require.Equal(t, uint64(4), lastProcessedId) - - lastReadLogID, err := ledgerStore.GetNextLogID(context.Background()) - require.NoError(t, err) - require.Equal(t, uint64(5), lastReadLogID) -} diff --git a/components/ledger/pkg/ledger/query/monitor.go b/components/ledger/pkg/ledger/query/monitor.go new file mode 100644 index 0000000000..121ecdf015 --- /dev/null +++ b/components/ledger/pkg/ledger/query/monitor.go @@ -0,0 +1,29 @@ +package query + +import ( + "context" + + "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/stack/libs/go-libs/metadata" +) + +type Monitor interface { + CommittedTransactions(ctx context.Context, res ...core.Transaction) + SavedMetadata(ctx context.Context, targetType, id string, metadata metadata.Metadata) + RevertedTransaction(ctx context.Context, reverted, revert *core.Transaction) +} + +type noOpMonitor struct{} + +func (n noOpMonitor) CommittedTransactions(ctx context.Context, res ...core.Transaction) { +} +func (n noOpMonitor) SavedMetadata(ctx context.Context, targetType string, id string, metadata metadata.Metadata) { +} +func (n noOpMonitor) RevertedTransaction(ctx context.Context, reverted, revert *core.Transaction) { +} + +var _ Monitor = &noOpMonitor{} + +func NewNoOpMonitor() *noOpMonitor { + return &noOpMonitor{} +} diff --git a/components/ledger/pkg/ledger/query/move_buffer.go b/components/ledger/pkg/ledger/query/move_buffer.go new file mode 100644 index 0000000000..175018783f --- /dev/null +++ b/components/ledger/pkg/ledger/query/move_buffer.go @@ -0,0 +1,125 @@ +package query + +import ( + "context" + "fmt" + "sync" + + "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/ledger/pkg/ledger/utils/job" + "github.com/formancehq/stack/libs/go-libs/collectionutils" +) + +type moveBufferInput struct { + move *core.Move + callback func() +} + +type moveBufferAccount struct { + account string + moves []*moveBufferInput +} + +type insertMovesJob struct { + buf *moveBuffer + moves []*moveBufferInput + accounts []*moveBufferAccount +} + +func (j insertMovesJob) String() string { + return fmt.Sprintf("inserting %d moves", len(j.moves)) +} + +func (j insertMovesJob) Terminated() { + for _, input := range j.moves { + input.callback() + } + + j.buf.mu.Lock() + defer j.buf.mu.Unlock() + for _, account := range j.accounts { + if len(account.moves) == 0 { + delete(j.buf.accounts, account.account) + } else { + j.buf.accountsQueue.Append(account) + } + } +} + +type moveBuffer struct { + *job.Runner[insertMovesJob] + accountsQueue *collectionutils.LinkedList[*moveBufferAccount] + accounts map[string]*moveBufferAccount + inputMoves chan *moveBufferInput + mu sync.Mutex + maxBufferSize int +} + +func (r *moveBuffer) AppendMove(move *core.Move, callback func()) { + r.mu.Lock() + mba, ok := r.accounts[move.Account] + if !ok { + mba = &moveBufferAccount{ + account: move.Account, + } + r.accounts[move.Account] = mba + r.accountsQueue.Append(mba) + } + mba.moves = append(mba.moves, &moveBufferInput{ + move: move, + callback: callback, + }) + r.mu.Unlock() + + r.Runner.Next() +} + +func (r *moveBuffer) nextJob() *insertMovesJob { + r.mu.Lock() + defer r.mu.Unlock() + + batch := make([]*moveBufferInput, 0) + accounts := make([]*moveBufferAccount, 0) + for { + mba := r.accountsQueue.TakeFirst() + if mba == nil { + break + } + accounts = append(accounts, mba) + + if len(batch)+len(mba.moves) >= r.maxBufferSize { + nbItems := r.maxBufferSize - len(batch) + batch = append(batch, mba.moves[:nbItems]...) + mba.moves = mba.moves[nbItems:] + break + } else { + batch = append(batch, mba.moves...) + mba.moves = make([]*moveBufferInput, 0) + } + } + + if len(batch) == 0 { + return nil + } + + return &insertMovesJob{ + accounts: accounts, + moves: batch, + buf: r, + } +} + +func newMoveBuffer(runner func(context.Context, ...*core.Move) error, nbWorkers, maxBufferSize int) *moveBuffer { + ret := &moveBuffer{ + accountsQueue: collectionutils.NewLinkedList[*moveBufferAccount](), + accounts: map[string]*moveBufferAccount{}, + inputMoves: make(chan *moveBufferInput), + maxBufferSize: maxBufferSize, + } + ret.Runner = job.NewJobRunner[insertMovesJob](func(ctx context.Context, job *insertMovesJob) error { + return runner(ctx, collectionutils.Map(job.moves, func(from *moveBufferInput) *core.Move { + return from.move + })...) + }, ret.nextJob, nbWorkers) + return ret +} diff --git a/components/ledger/pkg/ledger/query/move_buffer_test.go b/components/ledger/pkg/ledger/query/move_buffer_test.go new file mode 100644 index 0000000000..a295ee4f1b --- /dev/null +++ b/components/ledger/pkg/ledger/query/move_buffer_test.go @@ -0,0 +1,55 @@ +package query + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/stack/libs/go-libs/logging" +) + +func TestMoveBuffer(t *testing.T) { + t.Parallel() + + locked := sync.Map{} + buf := newMoveBuffer(func(ctx context.Context, moves ...*core.Move) error { + accounts := make(map[string]struct{}) + for _, move := range moves { + accounts[move.Account] = struct{}{} + } + for account := range accounts { + _, loaded := locked.LoadOrStore(account, struct{}{}) + if loaded { + panic(fmt.Sprintf("account '%s' already used", account)) + } + } + <-time.After(10 * time.Millisecond) + for account := range accounts { + locked.Delete(account) + } + + return nil + }, 5, 100) + go buf.Run(logging.ContextWithLogger(context.Background(), logging.Testing())) + defer buf.Close() + + wg := sync.WaitGroup{} + for i := 0; i < 100; i++ { + for j := 0; j < 100; j++ { + wg.Add(1) + j := j + go func() { + buf.AppendMove(&core.Move{ + Account: fmt.Sprintf("accounts:%d", j%10), + }, func() { + wg.Done() + }) + }() + } + } + wg.Wait() + <-time.After(time.Second) +} diff --git a/components/ledger/pkg/ledger/query/projector.go b/components/ledger/pkg/ledger/query/projector.go new file mode 100644 index 0000000000..c6e583d9df --- /dev/null +++ b/components/ledger/pkg/ledger/query/projector.go @@ -0,0 +1,260 @@ +package query + +import ( + "context" + "fmt" + "sync" + + "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/ledger/pkg/ledger/utils/batching" + "github.com/formancehq/ledger/pkg/opentelemetry/metrics" + storageerrors "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/stack/libs/go-libs/collectionutils" + "github.com/formancehq/stack/libs/go-libs/logging" +) + +type logPersistenceParts struct { + mu sync.Mutex + parts map[string]struct{} + onTerminated func() +} + +func (p *logPersistenceParts) Store(id string) { + p.mu.Lock() + defer p.mu.Unlock() + + p.parts[id] = struct{}{} +} + +func (p *logPersistenceParts) Delete(id string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.parts, id) + + if len(p.parts) == 0 { + p.onTerminated() + } +} + +func newLogPersistenceParts(onTerminated func()) *logPersistenceParts { + return &logPersistenceParts{ + mu: sync.Mutex{}, + parts: map[string]struct{}{}, + onTerminated: onTerminated, + } +} + +type Projector struct { + store Store + monitor Monitor + metricsRegistry metrics.PerLedgerRegistry + + queue chan []*core.ActiveLog + stopChan chan chan struct{} + activeLogs *collectionutils.LinkedList[*core.ActiveLog] + + txWorker *batching.Batcher[core.Transaction] + txMetadataWorker *batching.Batcher[core.TransactionWithMetadata] + accountMetadataWorker *batching.Batcher[core.Account] + + moveBuffer *moveBuffer + limitReadLogs int +} + +func (p *Projector) QueueLog(logs ...*core.ActiveLog) error { + p.queue <- logs + + return nil +} + +func (p *Projector) Stop(ctx context.Context) { + logger := logging.FromContext(ctx).WithField("component", "projector") + logger.Infof("Stop") + ch := make(chan struct{}) + p.stopChan <- ch + <-ch +} + +func (p *Projector) Start(ctx context.Context) { + logger := logging.FromContext(ctx).WithField("component", "projector") + logger.Infof("Start") + + ctx = logging.ContextWithLogger(ctx, logger) + + go p.moveBuffer.Run(logging.ContextWithField(ctx, "component", "moves buffer")) + go p.txWorker.Run(logging.ContextWithField(ctx, "component", "transactions buffer")) + go p.accountMetadataWorker.Run(logging.ContextWithField(ctx, "component", "accounts metadata buffer")) + go p.txMetadataWorker.Run(logging.ContextWithField(ctx, "component", "transactions metadata buffer")) + + p.syncLogs(ctx) + + go func() { + + for { + select { + case ch := <-p.stopChan: + logger.Debugf("Close move buffer") + p.moveBuffer.Close() + + logger.Debugf("Stop transaction worker") + p.txWorker.Close() + + logger.Debugf("Stop account metadata worker") + p.accountMetadataWorker.Close() + + logger.Debugf("Stop transaction metadata worker") + p.txMetadataWorker.Close() + + close(ch) + return + case logs := <-p.queue: + logger.Debugf("Got %d new logs to project", len(logs)) + p.processLogs(ctx, logs) + } + } + }() +} + +func (p *Projector) syncLogs(ctx context.Context) error { + lastReadLogID, err := p.store.GetNextLogID(ctx) + if err != nil && !storageerrors.IsNotFoundError(err) { + panic(err) + } + + logging.FromContext(ctx).Infof("Project logs since id: %d", lastReadLogID) + + for { + logs, err := p.store.ReadLogsRange(ctx, lastReadLogID, lastReadLogID+uint64(p.limitReadLogs)) + if err != nil { + panic(err) + } + + if len(logs) == 0 { + // No logs, nothing to do + return nil + } + + p.processLogs(ctx, collectionutils.Map(logs, func(from core.ChainedLog) *core.ActiveLog { + return core.NewActiveLog(&from) + })) + + lastReadLogID = logs[len(logs)-1].ID + 1 + + if len(logs) < p.limitReadLogs { + // Nothing to do anymore, no need to read more logs + return nil + } + } +} + +func (p *Projector) processLogs(ctx context.Context, logs []*core.ActiveLog) { + p.metricsRegistry.QueryInboundLogs().Add(ctx, int64(len(logs))) + p.activeLogs.Append(logs...) + + for _, log := range logs { + log := log + markLogAsProjected := func() { + log.SetProjected() + if err := p.store.MarkedLogsAsProjected(ctx, log.ID); err != nil { + panic(err) + } + p.metricsRegistry.QueryProcessedLogs().Add(ctx, 1) + } + dispatchTransaction := func(l *logPersistenceParts, log *core.ActiveLog, tx core.Transaction) { + logger := logging.FromContext(ctx).WithFields(map[string]any{ + "log-id": log.ID, + }) + moves := tx.GetMoves() + moveKey := func(move *core.Move) string { + return fmt.Sprintf("move/%d/%v/%s", move.PostingIndex, move.IsSource, move.Account) + } + l.Store("tx") + for _, move := range moves { + l.Store(moveKey(move)) + } + + p.txWorker.Append(tx, func() { + logger.Debugf("Transaction projected") + l.Delete("tx") + }) + + for _, move := range moves { + move := move + p.moveBuffer.AppendMove(move, func() { + logger.WithFields(map[string]any{ + "asset": move.Asset, + "is_source": move.IsSource, + "account": move.Account, + }).Debugf("Move projected") + l.Delete(moveKey(move)) + }) + } + } + switch payload := log.Log.Data.(type) { + case core.NewTransactionLogPayload: + l := newLogPersistenceParts(func() { + markLogAsProjected() + p.monitor.CommittedTransactions(ctx, *payload.Transaction) + }) + dispatchTransaction(l, log, *payload.Transaction) + case core.SetMetadataLogPayload: + switch payload.TargetType { + case core.MetaTargetTypeAccount: + p.accountMetadataWorker.Append(core.Account{ + Address: payload.TargetID.(string), + Metadata: payload.Metadata, + }, func() { + markLogAsProjected() + p.monitor.SavedMetadata(ctx, payload.TargetType, fmt.Sprint(payload.TargetID), payload.Metadata) + }) + case core.MetaTargetTypeTransaction: + p.txMetadataWorker.Append(core.TransactionWithMetadata{ + ID: payload.TargetID.(uint64), + Metadata: payload.Metadata, + }, func() { + markLogAsProjected() + p.monitor.SavedMetadata(ctx, payload.TargetType, fmt.Sprint(payload.TargetID), payload.Metadata) + }) + } + case core.RevertedTransactionLogPayload: + l := newLogPersistenceParts(func() { + markLogAsProjected() + p.activeLogs.RemoveValue(log) + + revertedTx, err := p.store.GetTransaction(ctx, payload.RevertedTransactionID) + if err != nil { + panic(err) + } + p.monitor.RevertedTransaction(ctx, payload.RevertTransaction, &revertedTx.Transaction) + }) + l.Store("metadata") + dispatchTransaction(l, log, *payload.RevertTransaction) + p.txMetadataWorker.Append(core.TransactionWithMetadata{ + ID: payload.RevertedTransactionID, + Metadata: core.RevertedMetadata(payload.RevertTransaction.ID), + }, func() { + l.Delete("metadata") + }) + } + } +} + +func NewProjector( + store Store, + monitor Monitor, + metricsRegistry metrics.PerLedgerRegistry, +) *Projector { + return &Projector{ + store: store, + monitor: monitor, + metricsRegistry: metricsRegistry, + txWorker: batching.NewBatcher(store.InsertTransactions, 2, 512), + accountMetadataWorker: batching.NewBatcher(store.UpdateAccountsMetadata, 1, 512), + txMetadataWorker: batching.NewBatcher(store.UpdateTransactionsMetadata, 1, 512), + moveBuffer: newMoveBuffer(store.InsertMoves, 5, 100), + activeLogs: collectionutils.NewLinkedList[*core.ActiveLog](), + queue: make(chan []*core.ActiveLog, 1024), + stopChan: make(chan chan struct{}), + limitReadLogs: 10000, + } +} diff --git a/components/ledger/pkg/ledger/query/projector_test.go b/components/ledger/pkg/ledger/query/projector_test.go new file mode 100644 index 0000000000..a7bbb87b0a --- /dev/null +++ b/components/ledger/pkg/ledger/query/projector_test.go @@ -0,0 +1,183 @@ +package query + +import ( + "context" + "fmt" + "math/big" + "testing" + "time" + + "github.com/alitto/pond" + "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/ledger/pkg/opentelemetry/metrics" + "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/formancehq/stack/libs/go-libs/metadata" + "github.com/stretchr/testify/require" +) + +func TestProjector(t *testing.T) { + t.Parallel() + + ledgerStore := storage.NewInMemoryStore() + + ctx := logging.TestingContext() + + projector := NewProjector(ledgerStore, NewNoOpMonitor(), metrics.NewNoOpRegistry()) + projector.Start(ctx) + defer projector.Stop(ctx) + + now := core.Now() + + tx0 := core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), + ) + tx1 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user:1", "USD/2", big.NewInt(10)), + ).WithID(1) + + appliedMetadataOnTX1 := metadata.Metadata{ + "paymentID": "1234", + } + appliedMetadataOnAccount := metadata.Metadata{ + "category": "gold", + } + + logs := []*core.ChainedLog{ + core.NewTransactionLog(tx0, nil).ChainLog(nil), + core.NewTransactionLog(tx1, nil).ChainLog(nil), + core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ + TargetType: core.MetaTargetTypeTransaction, + TargetID: tx1.ID, + Metadata: appliedMetadataOnTX1, + }).ChainLog(nil), + core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ + TargetType: core.MetaTargetTypeAccount, + TargetID: "bank", + Metadata: appliedMetadataOnAccount, + }).ChainLog(nil), + core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ + TargetType: core.MetaTargetTypeAccount, + TargetID: "another:account", + Metadata: appliedMetadataOnAccount, + }).ChainLog(nil), + } + for i, chainedLog := range logs { + chainedLog.ID = uint64(i) + activeLog := core.NewActiveLog(chainedLog) + require.NoError(t, projector.QueueLog(activeLog)) + <-activeLog.Projected + } + + ledgerStore.Logs = logs + require.Eventually(t, func() bool { + nextLogID, err := ledgerStore.GetNextLogID(context.Background()) + require.NoError(t, err) + return nextLogID == uint64(len(logs)) + }, time.Second, 100*time.Millisecond) + + require.EqualValues(t, 2, len(ledgerStore.Transactions)) + require.EqualValues(t, 4, len(ledgerStore.Accounts)) + + account := ledgerStore.Accounts["bank"] + require.NotNil(t, account) + require.NotEmpty(t, account.Volumes) + require.EqualValues(t, 100, account.Volumes["USD/2"].Input.Uint64()) + require.EqualValues(t, 10, account.Volumes["USD/2"].Output.Uint64()) + + tx1FromDatabase := ledgerStore.Transactions[1] + tx1.Metadata = appliedMetadataOnTX1 + require.Equal(t, core.ExpandedTransaction{ + Transaction: *tx1, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "bank": { + "USD/2": { + Input: big.NewInt(100), + Output: big.NewInt(0), + }, + }, + "user:1": { + "USD/2": { + Output: big.NewInt(0), + Input: big.NewInt(0), + }, + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "bank": { + "USD/2": { + Input: big.NewInt(100), + Output: big.NewInt(10), + }, + }, + "user:1": { + "USD/2": { + Input: big.NewInt(10), + Output: big.NewInt(0), + }, + }, + }, + }, *tx1FromDatabase) + + accountWithVolumes := ledgerStore.Accounts["bank"] + require.Equal(t, &core.AccountWithVolumes{ + Account: core.Account{ + Address: "bank", + Metadata: appliedMetadataOnAccount, + }, + Volumes: core.VolumesByAssets{ + "USD/2": { + Input: big.NewInt(100), + Output: big.NewInt(10), + }, + }, + }, accountWithVolumes) + + accountWithVolumes = ledgerStore.Accounts["another:account"] + require.Equal(t, &core.AccountWithVolumes{ + Account: core.Account{ + Address: "another:account", + Metadata: appliedMetadataOnAccount, + }, + Volumes: core.VolumesByAssets{}, + }, accountWithVolumes) +} + +func TestProjectorUnderHeavyParallelLoad(t *testing.T) { + t.Parallel() + + const nbWorkers = 5 + pool := pond.New(nbWorkers, nbWorkers) + ledgerStore := storage.NewInMemoryStore() + + ctx := logging.ContextWithLogger(context.TODO(), logging.Testing()) + + projector := NewProjector(ledgerStore, NewNoOpMonitor(), metrics.NewNoOpRegistry()) + projector.Start(ctx) + defer projector.Stop(ctx) + + var ( + previousLog *core.ChainedLog + allLogs = make([]*core.ActiveLog, 0) + ) + for i := 0; i < nbWorkers*500; i++ { + log := core.NewTransactionLog(core.NewTransaction().WithID(uint64(i)).WithPostings( + core.NewPosting("world", fmt.Sprintf("accounts:%d", i%100), "USD/2", big.NewInt(100)), + ), nil).ChainLog(previousLog) + activeLog := core.NewActiveLog(log) + pool.Submit(func() { + require.NoError(t, projector.QueueLog(activeLog)) + }) + previousLog = log + allLogs = append(allLogs, activeLog) + } + + pool.StopAndWait() + for _, log := range allLogs { + select { + case <-log.Projected: + case <-time.After(time.Second): + require.Fail(t, fmt.Sprintf("log %d must have been ingested", log.ID)) + } + } +} diff --git a/components/ledger/pkg/ledger/query/store.go b/components/ledger/pkg/ledger/query/store.go index 8267113892..6808045789 100644 --- a/components/ledger/pkg/ledger/query/store.go +++ b/components/ledger/pkg/ledger/query/store.go @@ -4,38 +4,17 @@ import ( "context" "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/storage/ledgerstore" ) type Store interface { - UpdateNextLogID(ctx context.Context, u uint64) error IsInitialized() bool GetNextLogID(ctx context.Context) (uint64, error) - ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core.PersistedLog, error) - RunInTransaction(ctx context.Context, f func(ctx context.Context, tx Store) error) error + ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core.ChainedLog, error) GetAccountWithVolumes(ctx context.Context, address string) (*core.AccountWithVolumes, error) GetTransaction(ctx context.Context, id uint64) (*core.ExpandedTransaction, error) - UpdateAccountsMetadata(ctx context.Context, update []core.Account) error - InsertTransactions(ctx context.Context, insert ...core.ExpandedTransaction) error + UpdateAccountsMetadata(ctx context.Context, update ...core.Account) error + InsertTransactions(ctx context.Context, insert ...core.Transaction) error + InsertMoves(ctx context.Context, insert ...*core.Move) error UpdateTransactionsMetadata(ctx context.Context, update ...core.TransactionWithMetadata) error - EnsureAccountsExist(ctx context.Context, accounts []string) error - UpdateVolumes(ctx context.Context, update ...core.AccountsAssetsVolumes) error -} - -type defaultStore struct { - *ledgerstore.Store -} - -func (d defaultStore) RunInTransaction(ctx context.Context, f func(ctx context.Context, tx Store) error) error { - return d.Store.RunInTransaction(ctx, func(ctx context.Context, store *ledgerstore.Store) error { - return f(ctx, NewDefaultStore(store)) - }) -} - -var _ Store = (*defaultStore)(nil) - -func NewDefaultStore(underlying *ledgerstore.Store) *defaultStore { - return &defaultStore{ - Store: underlying, - } + MarkedLogsAsProjected(ctx context.Context, id uint64) error } diff --git a/components/ledger/pkg/ledger/query/worker.go b/components/ledger/pkg/ledger/query/worker.go deleted file mode 100644 index 082b3b9608..0000000000 --- a/components/ledger/pkg/ledger/query/worker.go +++ /dev/null @@ -1,387 +0,0 @@ -package query - -import ( - "context" - "fmt" - - "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/ledger/aggregator" - "github.com/formancehq/ledger/pkg/ledger/monitor" - "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/stack/libs/go-libs/errorsutil" - "github.com/formancehq/stack/libs/go-libs/logging" - "github.com/formancehq/stack/libs/go-libs/metadata" - "github.com/pkg/errors" -) - -var ( - DefaultWorkerConfig = WorkerConfig{ - ChanSize: 1024, - } -) - -type WorkerConfig struct { - ChanSize int -} - -type logsData struct { - accountsToUpdate []core.Account - ensureAccountsExist []string - transactionsToInsert []core.ExpandedTransaction - transactionsToUpdate []core.TransactionWithMetadata - volumesToUpdate []core.AccountsAssetsVolumes - monitors []func(context.Context, monitor.Monitor) -} - -type Worker struct { - WorkerConfig - - pending []*ledgerstore.AppendedLog - writeChannel chan []*ledgerstore.AppendedLog - jobs chan []*ledgerstore.AppendedLog - stopChan chan chan struct{} - stoppedChan chan struct{} - writeLoopTerminated chan struct{} - readyChan chan struct{} - - store Store - monitor monitor.Monitor - metricsRegistry metrics.PerLedgerMetricsRegistry - - ledgerName string -} - -func (w *Worker) Ready() chan struct{} { - return w.readyChan -} - -func (w *Worker) Run(ctx context.Context) error { - logging.FromContext(ctx).Debugf("Start CQRS worker") - - close(w.readyChan) - - go w.writeLoop(ctx) - - stop := func(stopChan chan struct{}) { - close(w.stoppedChan) - select { - case <-ctx.Done(): - case <-w.writeLoopTerminated: - } - for _, log := range w.pending { - //TODO(gfyrag): forward an error - log.ActiveLog.SetIngested() - } - w.pending = nil - close(stopChan) - } - - var effectiveSendChannel chan []*ledgerstore.AppendedLog - for { - batch := w.pending - if len(batch) > 0 { - effectiveSendChannel = w.jobs - if len(batch) > 1024 { - batch = batch[:1024] - } - } else { - effectiveSendChannel = nil - } - select { - case <-ctx.Done(): - return ctx.Err() - - case stopChan := <-w.stopChan: - stop(stopChan) - return nil - - // At this level, the job is writing several models, just accumulate them in a buffer - case logs := <-w.writeChannel: - w.pending = append(w.pending, logs...) - w.metricsRegistry.QueryPendingMessages().Add(ctx, int64(len(w.pending))) - - case effectiveSendChannel <- batch: - w.pending = w.pending[len(batch):] - w.metricsRegistry.QueryPendingMessages().Add(ctx, int64(-len(batch))) - } - } -} - -func (w *Worker) writeLoop(ctx context.Context) { - closeLogs := func(logs []*ledgerstore.AppendedLog) { - for _, log := range logs { - close(log.ActiveLog.Ingested) - } - } - - for { - select { - case <-ctx.Done(): - return - case <-w.stoppedChan: - close(w.writeLoopTerminated) - return - case activeLogs := <-w.jobs: - logs := make([]core.PersistedLog, len(activeLogs)) - for i, holder := range activeLogs { - logs[i] = *holder.PersistedLog - } - - if err := processLogs(ctx, w.ledgerName, w.store, w.monitor, logs...); err != nil { - panic(err) - } - - w.metricsRegistry.QueryProcessedLogs().Add(ctx, int64(len(logs))) - - if err := w.store.UpdateNextLogID(ctx, logs[len(logs)-1].ID+1); err != nil { - panic(err) - } - - logging.FromContext(ctx).Debugf("Ingested logs until: %d", logs[len(logs)-1].ID) - closeLogs(activeLogs) - } - } -} - -func (w *Worker) Stop(ctx context.Context) error { - ch := make(chan struct{}) - select { - case <-ctx.Done(): - return ctx.Err() - case w.stopChan <- ch: - select { - case <-ctx.Done(): - return ctx.Err() - case <-ch: - } - } - - return nil -} - -func processLogs( - ctx context.Context, - ledgerName string, - store Store, - m monitor.Monitor, - logs ...core.PersistedLog, -) error { - logsData, err := buildData(ctx, ledgerName, store, logs...) - if err != nil { - return errors.Wrap(err, "building data") - } - - if err := store.RunInTransaction(ctx, func(ctx context.Context, tx Store) error { - if len(logsData.ensureAccountsExist) > 0 { - if err := tx.EnsureAccountsExist(ctx, logsData.ensureAccountsExist); err != nil { - return errors.Wrap(err, "ensuring accounts exist") - } - } - if len(logsData.accountsToUpdate) > 0 { - if err := tx.UpdateAccountsMetadata(ctx, logsData.accountsToUpdate); err != nil { - return errors.Wrap(err, "updating accounts metadata") - } - } - - if len(logsData.transactionsToInsert) > 0 { - if err := tx.InsertTransactions(ctx, logsData.transactionsToInsert...); err != nil { - return errors.Wrap(err, "inserting transactions") - } - } - - if len(logsData.transactionsToUpdate) > 0 { - if err := tx.UpdateTransactionsMetadata(ctx, logsData.transactionsToUpdate...); err != nil { - return errors.Wrap(err, "updating transactions") - } - } - - if len(logsData.volumesToUpdate) > 0 { - return tx.UpdateVolumes(ctx, logsData.volumesToUpdate...) - } - - return nil - }); err != nil { - return errorsutil.NewError(ErrStorage, err) - } - - if m != nil { - for _, monitor := range logsData.monitors { - monitor(ctx, m) - } - } - - return nil -} - -func buildData( - ctx context.Context, - ledgerName string, - store Store, - logs ...core.PersistedLog, -) (*logsData, error) { - logsData := &logsData{} - - volumeAggregator := aggregator.Volumes(store) - accountsToUpdate := make(map[string]metadata.Metadata) - transactionsToUpdate := make(map[uint64]metadata.Metadata) - - for _, log := range logs { - switch log.Type { - case core.NewTransactionLogType: - payload := log.Data.(core.NewTransactionLogPayload) - txVolumeAggregator, err := volumeAggregator.NextTxWithPostings(ctx, payload.Transaction.Postings...) - if err != nil { - return nil, err - } - - if payload.AccountMetadata != nil { - for account, metadata := range payload.AccountMetadata { - if m, ok := accountsToUpdate[account]; !ok { - accountsToUpdate[account] = metadata - } else { - for k, v := range metadata { - m[k] = v - } - } - } - } - - expandedTx := core.ExpandedTransaction{ - Transaction: *payload.Transaction, - PreCommitVolumes: txVolumeAggregator.PreCommitVolumes, - PostCommitVolumes: txVolumeAggregator.PostCommitVolumes, - } - - logsData.transactionsToInsert = append(logsData.transactionsToInsert, expandedTx) - - l: - for account, volumes := range txVolumeAggregator.PreCommitVolumes { - for _, volume := range volumes { - if volume.Output.Cmp(core.Zero) != 0 || volume.Input.Cmp(core.Zero) != 0 { - continue l - } - } - logsData.ensureAccountsExist = append(logsData.ensureAccountsExist, account) - } - - logsData.volumesToUpdate = append(logsData.volumesToUpdate, txVolumeAggregator.PostCommitVolumes) - - logsData.monitors = append(logsData.monitors, func(ctx context.Context, monitor monitor.Monitor) { - monitor.CommittedTransactions(ctx, ledgerName, expandedTx) - for account, metadata := range payload.AccountMetadata { - monitor.SavedMetadata(ctx, ledgerName, core.MetaTargetTypeAccount, account, metadata) - } - }) - - case core.SetMetadataLogType: - setMetadata := log.Data.(core.SetMetadataLogPayload) - switch setMetadata.TargetType { - case core.MetaTargetTypeAccount: - addr := setMetadata.TargetID.(string) - if m, ok := accountsToUpdate[addr]; !ok { - accountsToUpdate[addr] = setMetadata.Metadata - } else { - for k, v := range setMetadata.Metadata { - m[k] = v - } - } - - case core.MetaTargetTypeTransaction: - id := setMetadata.TargetID.(uint64) - if m, ok := transactionsToUpdate[id]; !ok { - transactionsToUpdate[id] = setMetadata.Metadata - } else { - for k, v := range setMetadata.Metadata { - m[k] = v - } - } - } - - logsData.monitors = append(logsData.monitors, func(ctx context.Context, monitor monitor.Monitor) { - monitor.SavedMetadata(ctx, ledgerName, setMetadata.TargetType, fmt.Sprint(setMetadata.TargetID), setMetadata.Metadata) - }) - - case core.RevertedTransactionLogType: - payload := log.Data.(core.RevertedTransactionLogPayload) - id := payload.RevertedTransactionID - metadata := core.RevertedMetadata(payload.RevertTransaction.ID) - if m, ok := transactionsToUpdate[id]; !ok { - transactionsToUpdate[id] = metadata - } else { - for k, v := range metadata { - m[k] = v - } - } - - txVolumeAggregator, err := volumeAggregator.NextTxWithPostings(ctx, payload.RevertTransaction.Postings...) - if err != nil { - return nil, errorsutil.NewError(ErrStorage, errors.Wrap(err, "aggregating volumes")) - } - - expandedTx := core.ExpandedTransaction{ - Transaction: *payload.RevertTransaction, - PreCommitVolumes: txVolumeAggregator.PreCommitVolumes, - PostCommitVolumes: txVolumeAggregator.PostCommitVolumes, - } - logsData.transactionsToInsert = append(logsData.transactionsToInsert, expandedTx) - - logsData.monitors = append(logsData.monitors, func(ctx context.Context, monitor monitor.Monitor) { - revertedTx, err := store.GetTransaction(ctx, payload.RevertedTransactionID) - if err != nil { - panic(err) - } - monitor.RevertedTransaction(ctx, ledgerName, revertedTx, &expandedTx) - }) - } - } - - for account, metadata := range accountsToUpdate { - logsData.accountsToUpdate = append(logsData.accountsToUpdate, core.Account{ - Address: account, - Metadata: metadata, - }) - } - - for transaction, metadata := range transactionsToUpdate { - logsData.transactionsToUpdate = append(logsData.transactionsToUpdate, core.TransactionWithMetadata{ - ID: transaction, - Metadata: metadata, - }) - } - - return logsData, nil -} - -func (w *Worker) QueueLog(ctx context.Context, logs ...*ledgerstore.AppendedLog) error { - select { - case <-w.stoppedChan: - return errors.New("worker stopped") - case w.writeChannel <- logs: - w.metricsRegistry.QueryInboundLogs().Add(ctx, 1) - return nil - } -} - -func NewWorker( - config WorkerConfig, - store Store, - ledgerName string, - monitor monitor.Monitor, - metricsRegistry metrics.PerLedgerMetricsRegistry, -) *Worker { - return &Worker{ - pending: make([]*ledgerstore.AppendedLog, 0, 1024), - jobs: make(chan []*ledgerstore.AppendedLog), - writeChannel: make(chan []*ledgerstore.AppendedLog, config.ChanSize), - stopChan: make(chan chan struct{}), - readyChan: make(chan struct{}), - stoppedChan: make(chan struct{}), - writeLoopTerminated: make(chan struct{}), - WorkerConfig: config, - store: store, - monitor: monitor, - ledgerName: ledgerName, - metricsRegistry: metricsRegistry, - } -} diff --git a/components/ledger/pkg/ledger/query/worker_test.go b/components/ledger/pkg/ledger/query/worker_test.go deleted file mode 100644 index 0c69bcd475..0000000000 --- a/components/ledger/pkg/ledger/query/worker_test.go +++ /dev/null @@ -1,268 +0,0 @@ -package query - -import ( - "context" - "math/big" - "testing" - "time" - - "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/ledger/monitor" - "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/stack/libs/go-libs/collectionutils" - "github.com/formancehq/stack/libs/go-libs/metadata" - "github.com/stretchr/testify/require" -) - -type mockStore struct { - nextLogID uint64 - logs []*core.PersistedLog - accounts map[string]*core.AccountWithVolumes - transactions []*core.ExpandedTransaction -} - -func (m *mockStore) UpdateAccountsMetadata(ctx context.Context, update []core.Account) error { - for _, account := range update { - persistedAccount, ok := m.accounts[account.Address] - if !ok { - m.accounts[account.Address] = &core.AccountWithVolumes{ - Account: account, - Volumes: map[string]core.Volumes{}, - } - return nil - } - persistedAccount.Metadata = persistedAccount.Metadata.Merge(account.Metadata) - } - return nil -} - -func (m *mockStore) InsertTransactions(ctx context.Context, insert ...core.ExpandedTransaction) error { - for _, transaction := range insert { - m.transactions = append(m.transactions, &transaction) - } - return nil -} - -func (m *mockStore) UpdateTransactionsMetadata(ctx context.Context, update ...core.TransactionWithMetadata) error { - for _, tx := range update { - m.transactions[tx.ID].Metadata = m.transactions[tx.ID].Metadata.Merge(tx.Metadata) - } - return nil -} - -func (m *mockStore) EnsureAccountsExist(ctx context.Context, accounts []string) error { - for _, address := range accounts { - _, ok := m.accounts[address] - if ok { - continue - } - m.accounts[address] = &core.AccountWithVolumes{ - Account: core.Account{ - Address: address, - Metadata: metadata.Metadata{}, - }, - Volumes: map[string]core.Volumes{}, - } - } - return nil -} - -func (m *mockStore) UpdateVolumes(ctx context.Context, updates ...core.AccountsAssetsVolumes) error { - for _, update := range updates { - for address, volumes := range update { - for asset, assetsVolumes := range volumes { - m.accounts[address].Volumes[asset] = assetsVolumes - } - } - } - return nil -} - -func (m *mockStore) UpdateNextLogID(ctx context.Context, id uint64) error { - m.nextLogID = id - return nil -} - -func (m *mockStore) IsInitialized() bool { - return true -} - -func (m *mockStore) GetNextLogID(ctx context.Context) (uint64, error) { - return m.nextLogID, nil -} - -func (m *mockStore) ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core.PersistedLog, error) { - if idMax > uint64(len(m.logs)) { - idMax = uint64(len(m.logs)) - } - - if idMin < uint64(len(m.logs)) { - return collectionutils.Map(m.logs[idMin:idMax], func(from *core.PersistedLog) core.PersistedLog { - return *from - }), nil - } - - return []core.PersistedLog{}, nil -} - -func (m *mockStore) RunInTransaction(ctx context.Context, f func(ctx context.Context, tx Store) error) error { - return f(ctx, m) -} - -func (m *mockStore) GetAccountWithVolumes(ctx context.Context, address string) (*core.AccountWithVolumes, error) { - account, ok := m.accounts[address] - if !ok { - return &core.AccountWithVolumes{ - Account: core.Account{ - Address: address, - Metadata: metadata.Metadata{}, - }, - Volumes: map[string]core.Volumes{}, - }, nil - } - return account, nil -} - -func (m *mockStore) GetTransaction(ctx context.Context, id uint64) (*core.ExpandedTransaction, error) { - return m.transactions[id], nil -} - -var _ Store = (*mockStore)(nil) - -func TestWorker(t *testing.T) { - t.Parallel() - - ledgerStore := &mockStore{ - accounts: map[string]*core.AccountWithVolumes{}, - } - - worker := NewWorker(WorkerConfig{ - ChanSize: 1024, - }, ledgerStore, "default", monitor.NewNoOpMonitor(), metrics.NewNoOpMetricsRegistry()) - go func() { - require.NoError(t, worker.Run(context.Background())) - }() - defer func() { - require.NoError(t, worker.Stop(context.Background())) - }() - <-worker.Ready() - - var ( - now = core.Now() - ) - - tx0 := core.NewTransaction().WithPostings( - core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), - ) - tx1 := core.NewTransaction().WithPostings( - core.NewPosting("bank", "user:1", "USD/2", big.NewInt(10)), - ).WithID(1) - - appliedMetadataOnTX1 := metadata.Metadata{ - "paymentID": "1234", - } - appliedMetadataOnAccount := metadata.Metadata{ - "category": "gold", - } - - logs := []*core.PersistedLog{ - core.NewTransactionLog(tx0, nil).ComputePersistentLog(nil), - core.NewTransactionLog(tx1, nil).ComputePersistentLog(nil), - core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeTransaction, - TargetID: tx1.ID, - Metadata: appliedMetadataOnTX1, - }).ComputePersistentLog(nil), - core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeAccount, - TargetID: "bank", - Metadata: appliedMetadataOnAccount, - }).ComputePersistentLog(nil), - core.NewSetMetadataLog(now, core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeAccount, - TargetID: "another:account", - Metadata: appliedMetadataOnAccount, - }).ComputePersistentLog(nil), - } - for i, persistedLog := range logs { - persistedLog.ID = uint64(i) - activeLog := core.NewActiveLog(&persistedLog.Log) - require.NoError(t, worker.QueueLog(context.Background(), &ledgerstore.AppendedLog{ - ActiveLog: activeLog, - PersistedLog: persistedLog, - })) - <-activeLog.Ingested - } - require.Eventually(t, func() bool { - nextLogID, err := ledgerStore.GetNextLogID(context.Background()) - require.NoError(t, err) - return nextLogID == uint64(len(logs)) - }, time.Second, 100*time.Millisecond) - - require.EqualValues(t, 2, len(ledgerStore.transactions)) - require.EqualValues(t, 4, len(ledgerStore.accounts)) - - account := ledgerStore.accounts["bank"] - require.NotNil(t, account) - require.NotEmpty(t, account.Volumes) - require.EqualValues(t, 100, account.Volumes["USD/2"].Input.Uint64()) - require.EqualValues(t, 10, account.Volumes["USD/2"].Output.Uint64()) - - tx1FromDatabase := ledgerStore.transactions[1] - tx1.Metadata = appliedMetadataOnTX1 - require.Equal(t, core.ExpandedTransaction{ - Transaction: *tx1, - PreCommitVolumes: map[string]core.AssetsVolumes{ - "bank": { - "USD/2": { - Input: big.NewInt(100), - Output: big.NewInt(0), - }, - }, - "user:1": { - "USD/2": { - Output: big.NewInt(0), - Input: big.NewInt(0), - }, - }, - }, - PostCommitVolumes: map[string]core.AssetsVolumes{ - "bank": { - "USD/2": { - Input: big.NewInt(100), - Output: big.NewInt(10), - }, - }, - "user:1": { - "USD/2": { - Input: big.NewInt(10), - Output: big.NewInt(0), - }, - }, - }, - }, *tx1FromDatabase) - - accountWithVolumes := ledgerStore.accounts["bank"] - require.Equal(t, &core.AccountWithVolumes{ - Account: core.Account{ - Address: "bank", - Metadata: appliedMetadataOnAccount, - }, - Volumes: map[string]core.Volumes{ - "USD/2": { - Input: big.NewInt(100), - Output: big.NewInt(10), - }, - }, - }, accountWithVolumes) - - accountWithVolumes = ledgerStore.accounts["another:account"] - require.Equal(t, &core.AccountWithVolumes{ - Account: core.Account{ - Address: "another:account", - Metadata: appliedMetadataOnAccount, - }, - Volumes: map[string]core.Volumes{}, - }, accountWithVolumes) -} diff --git a/components/ledger/pkg/ledger/resolver.go b/components/ledger/pkg/ledger/resolver.go index a87d79485f..b00527ecfc 100644 --- a/components/ledger/pkg/ledger/resolver.go +++ b/components/ledger/pkg/ledger/resolver.go @@ -4,25 +4,27 @@ import ( "context" "sync" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/formancehq/ledger/pkg/bus" "github.com/formancehq/ledger/pkg/ledger/command" - "github.com/formancehq/ledger/pkg/ledger/monitor" "github.com/formancehq/ledger/pkg/ledger/query" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) type option func(r *Resolver) -func WithMonitor(monitor monitor.Monitor) option { +func WithMessagePublisher(publisher message.Publisher) option { return func(r *Resolver) { - r.monitor = monitor + r.publisher = publisher } } -func WithMetricsRegistry(registry metrics.GlobalMetricsRegistry) option { +func WithMetricsRegistry(registry metrics.GlobalRegistry) option { return func(r *Resolver) { r.metricsRegistry = registry } @@ -34,23 +36,30 @@ func WithCompiler(compiler *command.Compiler) option { } } +func WithLogger(logger logging.Logger) option { + return func(r *Resolver) { + r.logger = logger + } +} + var defaultOptions = []option{ - WithMetricsRegistry(metrics.NewNoOpMetricsRegistry()), - WithMonitor(monitor.NewNoOpMonitor()), + WithMetricsRegistry(metrics.NewNoOpRegistry()), WithCompiler(command.NewCompiler(1024)), + WithLogger(logging.NewLogrus(logrus.New())), } type Resolver struct { - storageDriver *storage.Driver - monitor monitor.Monitor + storageDriver *driver.Driver lock sync.RWMutex - metricsRegistry metrics.GlobalMetricsRegistry + metricsRegistry metrics.GlobalRegistry //TODO(gfyrag): add a routine to clean old ledger - ledgers map[string]*Ledger - compiler *command.Compiler + ledgers map[string]*Ledger + compiler *command.Compiler + logger logging.Logger + publisher message.Publisher } -func NewResolver(storageDriver *storage.Driver, options ...option) *Resolver { +func NewResolver(storageDriver *driver.Driver, options ...option) *Resolver { r := &Resolver{ storageDriver: storageDriver, ledgers: map[string]*Ledger{}, @@ -70,6 +79,13 @@ func (r *Resolver) GetLedger(ctx context.Context, name string) (*Ledger, error) r.lock.Lock() defer r.lock.Unlock() + logging.FromContext(ctx).Infof("Initialize new ledger") + + ledger, ok = r.ledgers[name] + if ok { + return ledger, nil + } + exists, err := r.storageDriver.GetSystemStore().Exists(ctx, name) if err != nil { return nil, err @@ -91,18 +107,6 @@ func (r *Resolver) GetLedger(ctx context.Context, name string) (*Ledger, error) } } - backgroundContext := logging.ContextWithLogger( - context.Background(), - logging.FromContext(ctx), - ) - runOrPanic := func(task func(context.Context) error) { - go func() { - if err := task(backgroundContext); err != nil { - panic(err) - } - }() - } - locker := command.NewDefaultLocker() metricsRegistry, err := metrics.RegisterPerLedgerMetricsRegistry(name) @@ -110,10 +114,15 @@ func (r *Resolver) GetLedger(ctx context.Context, name string) (*Ledger, error) return nil, errors.Wrap(err, "registering metrics") } - queryWorker := query.NewWorker(query.DefaultWorkerConfig, query.NewDefaultStore(store), name, r.monitor, metricsRegistry) - runOrPanic(queryWorker.Run) + var monitor query.Monitor = query.NewNoOpMonitor() + if r.publisher != nil { + monitor = bus.NewLedgerMonitor(r.publisher, name) + } + + projector := query.NewProjector(store, monitor, metricsRegistry) + projector.Start(logging.ContextWithLogger(context.Background(), r.logger)) - ledger = New(store, locker, queryWorker, r.compiler, metricsRegistry) + ledger = New(store, locker, projector, r.compiler, metricsRegistry) r.ledgers[name] = ledger r.metricsRegistry.ActiveLedgers().Add(ctx, +1) } @@ -122,8 +131,13 @@ func (r *Resolver) GetLedger(ctx context.Context, name string) (*Ledger, error) } func (r *Resolver) CloseLedgers(ctx context.Context) error { + r.logger.Info("Close all ledgers") + defer func() { + r.logger.Info("All ledgers closed") + }() for name, ledger := range r.ledgers { - if err := ledger.Close(ctx); err != nil { + r.logger.Infof("Close ledger %s", name) + if err := ledger.Close(logging.ContextWithLogger(ctx, r.logger.WithField("ledger", name))); err != nil { return err } delete(r.ledgers, name) diff --git a/components/ledger/pkg/ledger/utils/batching/batcher.go b/components/ledger/pkg/ledger/utils/batching/batcher.go new file mode 100644 index 0000000000..0ff54cbde5 --- /dev/null +++ b/components/ledger/pkg/ledger/utils/batching/batcher.go @@ -0,0 +1,75 @@ +package batching + +import ( + "context" + "fmt" + "sync" + + "github.com/formancehq/ledger/pkg/ledger/utils/job" + "github.com/formancehq/stack/libs/go-libs/collectionutils" +) + +type pending[T any] struct { + object T + callback func() +} + +type batcherJob[T any] []*pending[T] + +func (b batcherJob[T]) String() string { + return fmt.Sprintf("processing %d items", len(b)) +} + +func (b batcherJob[T]) Terminated() { + for _, v := range b { + v.callback() + } +} + +type Batcher[T any] struct { + *job.Runner[batcherJob[T]] + pending []*pending[T] + mu sync.Mutex + maxBatchSize int +} + +func (s *Batcher[T]) Append(object T, callback func()) { + s.mu.Lock() + s.pending = append(s.pending, &pending[T]{ + callback: callback, + object: object, + }) + s.mu.Unlock() + s.Runner.Next() +} + +func (s *Batcher[T]) nextBatch() *batcherJob[T] { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.pending) == 0 { + return nil + } + if len(s.pending) > s.maxBatchSize { + batch := s.pending[:s.maxBatchSize] + s.pending = s.pending[s.maxBatchSize:] + ret := batcherJob[T](batch) + return &ret + } + batch := s.pending + s.pending = make([]*pending[T], 0) + ret := batcherJob[T](batch) + return &ret +} + +func NewBatcher[T any](runner func(context.Context, ...T) error, nbWorkers, maxBatchSize int) *Batcher[T] { + ret := &Batcher[T]{ + maxBatchSize: maxBatchSize, + } + ret.Runner = job.NewJobRunner[batcherJob[T]](func(ctx context.Context, job *batcherJob[T]) error { + return runner(ctx, collectionutils.Map(*job, func(from *pending[T]) T { + return from.object + })...) + }, ret.nextBatch, nbWorkers) + return ret +} diff --git a/components/ledger/pkg/ledger/utils/job/jobs.go b/components/ledger/pkg/ledger/utils/job/jobs.go new file mode 100644 index 0000000000..dcd599c9f9 --- /dev/null +++ b/components/ledger/pkg/ledger/utils/job/jobs.go @@ -0,0 +1,134 @@ +package job + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/alitto/pond" + "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/pkg/errors" +) + +type Job interface { + Terminated() +} + +type builtJob struct { + terminatedFn func() +} + +func (j builtJob) Terminated() { + j.terminatedFn() +} + +func newJob(terminatedFn func()) *builtJob { + return &builtJob{ + terminatedFn: terminatedFn, + } +} + +type Runner[JOB Job] struct { + stopChan chan chan struct{} + runner func(context.Context, *JOB) error + nbWorkers int + parkedWorkers atomic.Int64 + nextJob func() *JOB + jobs chan *JOB + newJobsAvailable chan struct{} +} + +func (r *Runner[JOB]) Next() { + r.newJobsAvailable <- struct{}{} +} + +func (r *Runner[JOB]) Close() { + done := make(chan struct{}) + r.stopChan <- done + <-done +} + +func (r *Runner[JOB]) Run(ctx context.Context) { + + logger := logging.FromContext(ctx) + logger.Infof("Start worker") + + terminatedJobs := make(chan *JOB, r.nbWorkers) + jobsErrors := make(chan error, r.nbWorkers) + + w := pond.New(r.nbWorkers, r.nbWorkers) + for i := 0; i < r.nbWorkers; i++ { + i := i + w.Submit(func() { + defer func() { + if e := recover(); e != nil { + if err, isError := e.(error); isError { + jobsErrors <- errors.WithStack(err) + return + } + jobsErrors <- errors.WithStack(fmt.Errorf("%s", e)) + } + }() + logger := logger.WithFields(map[string]any{ + "worker": i, + }) + for { + select { + case job, ok := <-r.jobs: + if !ok { + logger.Debugf("Worker %d stopped", i) + return + } + logger := logger.WithField("job", job) + logger.Debugf("Got new job") + if err := r.runner(ctx, job); err != nil { + panic(err) + } + logger.Debugf("Job terminated") + terminatedJobs <- job + } + } + }) + } + + for { + select { + case jobError := <-jobsErrors: + panic(jobError) + case done := <-r.stopChan: + close(r.jobs) + w.StopAndWait() + close(terminatedJobs) + close(done) + return + case <-r.newJobsAvailable: + if r.parkedWorkers.Load() > 0 { + if job := r.nextJob(); job != nil { + r.jobs <- job + r.parkedWorkers.Add(-1) + } + } + case job := <-terminatedJobs: + (*job).Terminated() + if job := r.nextJob(); job != nil { + r.jobs <- job + } else { + r.parkedWorkers.Add(1) + } + } + } +} + +func NewJobRunner[JOB Job](runner func(context.Context, *JOB) error, nextJob func() *JOB, nbWorkers int) *Runner[JOB] { + parkedWorkers := atomic.Int64{} + parkedWorkers.Add(int64(nbWorkers)) + return &Runner[JOB]{ + stopChan: make(chan chan struct{}), + runner: runner, + nbWorkers: nbWorkers, + parkedWorkers: parkedWorkers, + nextJob: nextJob, + jobs: make(chan *JOB, nbWorkers), + newJobsAvailable: make(chan struct{}), + } +} diff --git a/components/ledger/pkg/ledger/utils/job/jobs_test.go b/components/ledger/pkg/ledger/utils/job/jobs_test.go new file mode 100644 index 0000000000..1b875d71a5 --- /dev/null +++ b/components/ledger/pkg/ledger/utils/job/jobs_test.go @@ -0,0 +1,44 @@ +package job + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/stretchr/testify/require" +) + +func TestWorkerPool(t *testing.T) { + t.Parallel() + + const countJobs = 10000 + createdJobs := atomic.Int64{} + terminatedJobs := atomic.Int64{} + nextJob := func() *builtJob { + if createdJobs.Load() == 10000 { + return nil + } + createdJobs.Add(1) + return newJob(func() { + terminatedJobs.Add(1) + }) + } + runner := func(ctx context.Context, job *builtJob) error { + return nil + } + ctx := logging.TestingContext() + + pool := NewJobRunner[builtJob](runner, nextJob, 5) + go pool.Run(ctx) + defer pool.Close() + + for i := 0; i < 100; i++ { + go pool.Next() // Simulate random input + } + + require.Eventually(t, func() bool { + return countJobs == createdJobs.Load() + }, 5*time.Second, time.Millisecond*100) +} diff --git a/components/ledger/pkg/machine/vm/store.go b/components/ledger/pkg/machine/vm/store.go index d7400036e5..a78b019147 100644 --- a/components/ledger/pkg/machine/vm/store.go +++ b/components/ledger/pkg/machine/vm/store.go @@ -5,7 +5,7 @@ import ( "math/big" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" ) type Store interface { diff --git a/components/ledger/pkg/opentelemetry/metrics/metrics.go b/components/ledger/pkg/opentelemetry/metrics/metrics.go index 736b297184..1e8220048f 100644 --- a/components/ledger/pkg/opentelemetry/metrics/metrics.go +++ b/components/ledger/pkg/opentelemetry/metrics/metrics.go @@ -6,20 +6,20 @@ import ( "go.opentelemetry.io/otel/metric/instrument" ) -type GlobalMetricsRegistry interface { +type GlobalRegistry interface { APILatencies() instrument.Int64Histogram StatusCodes() instrument.Int64Counter ActiveLedgers() instrument.Int64UpDownCounter } -type globalMetricsRegistry struct { +type globalRegistry struct { // API Latencies - aPILatencies instrument.Int64Histogram + apiLatencies instrument.Int64Histogram statusCodes instrument.Int64Counter activeLedgers instrument.Int64UpDownCounter } -func RegisterGlobalMetricsRegistry(meterProvider metric.MeterProvider) (GlobalMetricsRegistry, error) { +func RegisterGlobalRegistry(meterProvider metric.MeterProvider) (GlobalRegistry, error) { meter := meterProvider.Meter("global") apiLatencies, err := meter.Int64Histogram( @@ -49,26 +49,26 @@ func RegisterGlobalMetricsRegistry(meterProvider metric.MeterProvider) (GlobalMe return nil, err } - return &globalMetricsRegistry{ - aPILatencies: apiLatencies, + return &globalRegistry{ + apiLatencies: apiLatencies, statusCodes: statusCodes, activeLedgers: activeLedgers, }, nil } -func (gm *globalMetricsRegistry) APILatencies() instrument.Int64Histogram { - return gm.aPILatencies +func (gm *globalRegistry) APILatencies() instrument.Int64Histogram { + return gm.apiLatencies } -func (gm *globalMetricsRegistry) StatusCodes() instrument.Int64Counter { +func (gm *globalRegistry) StatusCodes() instrument.Int64Counter { return gm.statusCodes } -func (gm *globalMetricsRegistry) ActiveLedgers() instrument.Int64UpDownCounter { +func (gm *globalRegistry) ActiveLedgers() instrument.Int64UpDownCounter { return gm.activeLedgers } -type PerLedgerMetricsRegistry interface { +type PerLedgerRegistry interface { CacheMisses() instrument.Int64Counter CacheNumberEntries() instrument.Int64UpDownCounter @@ -78,7 +78,7 @@ type PerLedgerMetricsRegistry interface { QueryProcessedLogs() instrument.Int64Counter } -type perLedgerMetricsRegistry struct { +type perLedgerRegistry struct { cacheMisses instrument.Int64Counter cacheNumberEntries instrument.Int64UpDownCounter @@ -88,7 +88,7 @@ type perLedgerMetricsRegistry struct { queryProcessedLogs instrument.Int64Counter } -func RegisterPerLedgerMetricsRegistry(ledger string) (PerLedgerMetricsRegistry, error) { +func RegisterPerLedgerMetricsRegistry(ledger string) (PerLedgerRegistry, error) { // we can now use the global meter provider to create a meter // since it was created by the fx meter := global.MeterProvider().Meter(ledger) @@ -147,7 +147,7 @@ func RegisterPerLedgerMetricsRegistry(ledger string) (PerLedgerMetricsRegistry, return nil, err } - return &perLedgerMetricsRegistry{ + return &perLedgerRegistry{ cacheMisses: cacheMisses, cacheNumberEntries: cacheNumberEntries, queryLatencies: queryLatencies, @@ -157,77 +157,77 @@ func RegisterPerLedgerMetricsRegistry(ledger string) (PerLedgerMetricsRegistry, }, nil } -func (pm *perLedgerMetricsRegistry) CacheMisses() instrument.Int64Counter { +func (pm *perLedgerRegistry) CacheMisses() instrument.Int64Counter { return pm.cacheMisses } -func (pm *perLedgerMetricsRegistry) CacheNumberEntries() instrument.Int64UpDownCounter { +func (pm *perLedgerRegistry) CacheNumberEntries() instrument.Int64UpDownCounter { return pm.cacheNumberEntries } -func (pm *perLedgerMetricsRegistry) QueryLatencies() instrument.Int64Histogram { +func (pm *perLedgerRegistry) QueryLatencies() instrument.Int64Histogram { return pm.queryLatencies } -func (pm *perLedgerMetricsRegistry) QueryInboundLogs() instrument.Int64Counter { +func (pm *perLedgerRegistry) QueryInboundLogs() instrument.Int64Counter { return pm.queryInboundLogs } -func (pm *perLedgerMetricsRegistry) QueryPendingMessages() instrument.Int64Counter { +func (pm *perLedgerRegistry) QueryPendingMessages() instrument.Int64Counter { return pm.queryPendingMessages } -func (pm *perLedgerMetricsRegistry) QueryProcessedLogs() instrument.Int64Counter { +func (pm *perLedgerRegistry) QueryProcessedLogs() instrument.Int64Counter { return pm.queryProcessedLogs } -type NoOpMetricsRegistry struct{} +type noOpRegistry struct{} -func NewNoOpMetricsRegistry() *NoOpMetricsRegistry { - return &NoOpMetricsRegistry{} +func NewNoOpRegistry() *noOpRegistry { + return &noOpRegistry{} } -func (nm *NoOpMetricsRegistry) CacheMisses() instrument.Int64Counter { +func (nm *noOpRegistry) CacheMisses() instrument.Int64Counter { counter, _ := metric.NewNoopMeter().Int64Counter("cache_misses") return counter } -func (nm *NoOpMetricsRegistry) CacheNumberEntries() instrument.Int64UpDownCounter { +func (nm *noOpRegistry) CacheNumberEntries() instrument.Int64UpDownCounter { counter, _ := metric.NewNoopMeter().Int64UpDownCounter("cache_number_entries") return counter } -func (nm *NoOpMetricsRegistry) QueryLatencies() instrument.Int64Histogram { +func (nm *noOpRegistry) QueryLatencies() instrument.Int64Histogram { histogram, _ := metric.NewNoopMeter().Int64Histogram("query_latencies") return histogram } -func (nm *NoOpMetricsRegistry) QueryInboundLogs() instrument.Int64Counter { +func (nm *noOpRegistry) QueryInboundLogs() instrument.Int64Counter { counter, _ := metric.NewNoopMeter().Int64Counter("query_inbound_logs") return counter } -func (nm *NoOpMetricsRegistry) QueryPendingMessages() instrument.Int64Counter { +func (nm *noOpRegistry) QueryPendingMessages() instrument.Int64Counter { counter, _ := metric.NewNoopMeter().Int64Counter("query_pending_messages") return counter } -func (nm *NoOpMetricsRegistry) QueryProcessedLogs() instrument.Int64Counter { +func (nm *noOpRegistry) QueryProcessedLogs() instrument.Int64Counter { counter, _ := metric.NewNoopMeter().Int64Counter("query_processed_logs") return counter } -func (nm *NoOpMetricsRegistry) APILatencies() instrument.Int64Histogram { +func (nm *noOpRegistry) APILatencies() instrument.Int64Histogram { histogram, _ := metric.NewNoopMeter().Int64Histogram("api_latencies") return histogram } -func (nm *NoOpMetricsRegistry) StatusCodes() instrument.Int64Counter { +func (nm *noOpRegistry) StatusCodes() instrument.Int64Counter { counter, _ := metric.NewNoopMeter().Int64Counter("status_codes") return counter } -func (nm *NoOpMetricsRegistry) ActiveLedgers() instrument.Int64UpDownCounter { +func (nm *noOpRegistry) ActiveLedgers() instrument.Int64UpDownCounter { counter, _ := metric.NewNoopMeter().Int64UpDownCounter("active_ledgers") return counter } diff --git a/components/ledger/pkg/storage/database.go b/components/ledger/pkg/storage/database.go new file mode 100644 index 0000000000..c78d5851cd --- /dev/null +++ b/components/ledger/pkg/storage/database.go @@ -0,0 +1,40 @@ +package storage + +import ( + "context" + + "github.com/uptrace/bun" +) + +type Database struct { + db *bun.DB +} + +func (p *Database) Initialize(ctx context.Context) error { + _, err := p.db.ExecContext(ctx, "CREATE EXTENSION IF NOT EXISTS pgcrypto") + if err != nil { + return PostgresError(err) + } + _, err = p.db.ExecContext(ctx, "CREATE EXTENSION IF NOT EXISTS pg_trgm") + if err != nil { + return PostgresError(err) + } + return nil +} + +func (p *Database) Schema(name string) (Schema, error) { + return Schema{ + IDB: p.db, + name: name, + }, nil +} + +func (p *Database) Close(ctx context.Context) error { + return p.db.Close() +} + +func NewDatabase(db *bun.DB) *Database { + return &Database{ + db: db, + } +} diff --git a/components/ledger/pkg/storage/cli.go b/components/ledger/pkg/storage/driver/cli.go similarity index 83% rename from components/ledger/pkg/storage/cli.go rename to components/ledger/pkg/storage/driver/cli.go index 7275f7e90b..f7bdcae0cd 100644 --- a/components/ledger/pkg/storage/cli.go +++ b/components/ledger/pkg/storage/driver/cli.go @@ -1,12 +1,11 @@ -package storage +package driver import ( "io" "time" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/ledger/pkg/storage/schema" - "github.com/formancehq/ledger/pkg/storage/utils" "github.com/formancehq/stack/libs/go-libs/health" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -40,16 +39,16 @@ type PostgresConfig struct { } type ModuleConfig struct { - PostgresConnectionOptions utils.ConnectionOptions + PostgresConnectionOptions storage.ConnectionOptions StoreConfig ledgerstore.StoreConfig Debug bool } -func CLIDriverModule(v *viper.Viper, output io.Writer, debug bool) fx.Option { +func CLIModule(v *viper.Viper, output io.Writer, debug bool) fx.Option { options := make([]fx.Option, 0) options = append(options, fx.Provide(func() (*bun.DB, error) { - return utils.OpenSQLDB(utils.ConnectionOptions{ + return storage.OpenSQLDB(storage.ConnectionOptions{ DatabaseSourceName: v.GetString(StoragePostgresConnectionStringFlag), Debug: debug, Writer: output, @@ -58,11 +57,11 @@ func CLIDriverModule(v *viper.Viper, output io.Writer, debug bool) fx.Option { MaxOpenConns: v.GetInt(StoragePostgresMaxOpenConns), }) })) - options = append(options, fx.Provide(func(db *bun.DB) schema.DB { - return schema.NewPostgresDB(db) + options = append(options, fx.Provide(func(db *bun.DB) *storage.Database { + return storage.NewDatabase(db) })) - options = append(options, fx.Provide(func(db schema.DB) (*Driver, error) { - return NewDriver("postgres", db, ledgerstore.StoreConfig{ + options = append(options, fx.Provide(func(db *storage.Database) (*Driver, error) { + return New("postgres", db, ledgerstore.StoreConfig{ StoreWorkerConfig: ledgerstore.Config{ MaxPendingSize: v.GetInt(StoreWorkerMaxPendingSize), MaxWriteChanSize: v.GetInt(StoreWorkerMaxWriteChanSize), diff --git a/components/ledger/pkg/storage/driver.go b/components/ledger/pkg/storage/driver/driver.go similarity index 88% rename from components/ledger/pkg/storage/driver.go rename to components/ledger/pkg/storage/driver/driver.go index 02b612f41c..6edae8c876 100644 --- a/components/ledger/pkg/storage/driver.go +++ b/components/ledger/pkg/storage/driver/driver.go @@ -1,4 +1,4 @@ -package storage +package driver import ( "context" @@ -7,9 +7,8 @@ import ( "fmt" "sync" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/ledger/pkg/storage/schema" systemstore "github.com/formancehq/ledger/pkg/storage/systemstore" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/pkg/errors" @@ -67,7 +66,7 @@ func InstrumentalizeSQLDriver() { type Driver struct { name string - db schema.DB + db *storage.Database systemStore *systemstore.Store lock sync.Mutex storeConfig ledgerstore.StoreConfig @@ -87,13 +86,13 @@ func (d *Driver) newStore(ctx context.Context, name string) (*ledgerstore.Store, return nil, err } - store, err := ledgerstore.NewStore(schema, func(ctx context.Context) error { + store, err := ledgerstore.New(schema, func(ctx context.Context) error { return d.GetSystemStore().DeleteLedger(ctx, name) }, d.storeConfig) if err != nil { return nil, err } - go store.Run(context.Background()) + go store.Run(logging.ContextWithLogger(context.Background(), logging.FromContext(ctx).WithField("component", "store"))) return store, nil } @@ -110,7 +109,7 @@ func (d *Driver) CreateLedgerStore(ctx context.Context, name string) (*ledgersto return nil, err } if exists { - return nil, storageerrors.ErrStoreAlreadyExists + return nil, storage.ErrStoreAlreadyExists } _, err = d.systemStore.Register(ctx, name) @@ -137,7 +136,7 @@ func (d *Driver) GetLedgerStore(ctx context.Context, name string) (*ledgerstore. return nil, errors.Wrap(err, "checking ledger existence") } if !exists { - return nil, storageerrors.ErrStoreNotFound + return nil, storage.ErrStoreNotFound } return d.newStore(ctx, name) @@ -176,7 +175,7 @@ func (d *Driver) Close(ctx context.Context) error { return d.db.Close(ctx) } -func NewDriver(name string, db schema.DB, storeConfig ledgerstore.StoreConfig) *Driver { +func New(name string, db *storage.Database, storeConfig ledgerstore.StoreConfig) *Driver { return &Driver{ db: db, name: name, diff --git a/components/ledger/pkg/storage/driver_test.go b/components/ledger/pkg/storage/driver/driver_test.go similarity index 76% rename from components/ledger/pkg/storage/driver_test.go rename to components/ledger/pkg/storage/driver/driver_test.go index 515f9addc1..c06da668d0 100644 --- a/components/ledger/pkg/storage/driver_test.go +++ b/components/ledger/pkg/storage/driver/driver_test.go @@ -1,12 +1,12 @@ -package storage_test +package driver_test import ( "context" "os" "testing" - "github.com/formancehq/ledger/pkg/storage/errors" - "github.com/formancehq/ledger/pkg/storage/sqlstoragetesting" + "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/storagetesting" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" @@ -25,7 +25,7 @@ func TestMain(t *testing.M) { } func TestConfiguration(t *testing.T) { - d := sqlstoragetesting.StorageDriver(t) + d := storagetesting.StorageDriver(t) require.NoError(t, d.GetSystemStore().InsertConfiguration(context.Background(), "foo", "bar")) bar, err := d.GetSystemStore().GetConfiguration(context.Background(), "foo") @@ -34,9 +34,9 @@ func TestConfiguration(t *testing.T) { } func TestConfigurationError(t *testing.T) { - d := sqlstoragetesting.StorageDriver(t) + d := storagetesting.StorageDriver(t) _, err := d.GetSystemStore().GetConfiguration(context.Background(), "not_existing") require.Error(t, err) - require.True(t, errors.IsNotFoundError(err)) + require.True(t, storage.IsNotFoundError(err)) } diff --git a/components/ledger/pkg/storage/errors/errors.go b/components/ledger/pkg/storage/errors.go similarity index 64% rename from components/ledger/pkg/storage/errors/errors.go rename to components/ledger/pkg/storage/errors.go index ba38900b4f..3e7eeb9a1d 100644 --- a/components/ledger/pkg/storage/errors/errors.go +++ b/components/ledger/pkg/storage/errors.go @@ -1,4 +1,4 @@ -package errors +package storage import ( "database/sql" @@ -33,22 +33,12 @@ func PostgresError(err error) error { return nil } -func StorageError(err error) error { - if err == nil { - return nil - } - - return errorsutil.NewError(ErrStorage, err) -} - -var ErrNotFound = errors.New("not found") - var ( - ErrConstraintFailed = errors.New("23505: constraint failed") - ErrTooManyClients = errors.New("53300: too many clients") - ErrStoreNotInitialized = errors.New("store not initialized") - ErrStoreAlreadyExists = errors.New("store already exists") - ErrStoreNotFound = errors.New("store not found") + ErrNotFound = errors.New("not found") + ErrConstraintFailed = errors.New("23505: constraint failed") + ErrTooManyClients = errors.New("53300: too many clients") + ErrStoreAlreadyExists = errors.New("store already exists") + ErrStoreNotFound = errors.New("store not found") ErrStorage = errors.New("storage error") ) diff --git a/components/ledger/pkg/storage/inmemory.go b/components/ledger/pkg/storage/inmemory.go new file mode 100644 index 0000000000..91b0379d72 --- /dev/null +++ b/components/ledger/pkg/storage/inmemory.go @@ -0,0 +1,189 @@ +package storage + +import ( + "context" + + "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/stack/libs/go-libs/collectionutils" + "github.com/formancehq/stack/libs/go-libs/metadata" +) + +type InMemoryStore struct { + Logs []*core.ChainedLog + Accounts map[string]*core.AccountWithVolumes + Transactions []*core.ExpandedTransaction +} + +func (m *InMemoryStore) MarkedLogsAsProjected(ctx context.Context, id uint64) error { + for _, log := range m.Logs { + if log.ID == id { + log.Projected = true + return nil + } + } + return nil +} + +func (m *InMemoryStore) InsertMoves(ctx context.Context, insert ...*core.Move) error { + // TODO(gfyrag): to reflect the behavior of the real storage, we should compute accounts volumes there + return nil +} + +func (m *InMemoryStore) UpdateAccountsMetadata(ctx context.Context, update ...core.Account) error { + for _, account := range update { + persistedAccount, ok := m.Accounts[account.Address] + if !ok { + m.Accounts[account.Address] = &core.AccountWithVolumes{ + Account: account, + Volumes: core.VolumesByAssets{}, + } + return nil + } + persistedAccount.Metadata = persistedAccount.Metadata.Merge(account.Metadata) + } + return nil +} + +func (m *InMemoryStore) InsertTransactions(ctx context.Context, insert ...core.Transaction) error { + for _, transaction := range insert { + expandedTransaction := &core.ExpandedTransaction{ + Transaction: transaction, + PreCommitVolumes: core.AccountsAssetsVolumes{}, + PostCommitVolumes: core.AccountsAssetsVolumes{}, + } + for _, posting := range transaction.Postings { + account, ok := m.Accounts[posting.Source] + if !ok { + account = core.NewAccountWithVolumes(posting.Source) + m.Accounts[posting.Source] = account + } + + asset, ok := account.Volumes[posting.Asset] + if !ok { + asset = core.NewEmptyVolumes() + account.Volumes[posting.Asset] = asset + } + + account, ok = m.Accounts[posting.Destination] + if !ok { + account = core.NewAccountWithVolumes(posting.Destination) + m.Accounts[posting.Destination] = account + } + + asset, ok = account.Volumes[posting.Asset] + if !ok { + asset = core.NewEmptyVolumes() + account.Volumes[posting.Asset] = asset + } + } + for _, posting := range transaction.Postings { + expandedTransaction.PreCommitVolumes.AddOutput(posting.Source, posting.Asset, + m.Accounts[posting.Source].Volumes[posting.Asset].Output) + expandedTransaction.PreCommitVolumes.AddInput(posting.Source, posting.Asset, + m.Accounts[posting.Source].Volumes[posting.Asset].Input) + + expandedTransaction.PreCommitVolumes.AddOutput(posting.Destination, posting.Asset, + m.Accounts[posting.Destination].Volumes[posting.Asset].Output) + expandedTransaction.PreCommitVolumes.AddInput(posting.Destination, posting.Asset, + m.Accounts[posting.Destination].Volumes[posting.Asset].Input) + } + for _, posting := range transaction.Postings { + account := m.Accounts[posting.Source] + asset := account.Volumes[posting.Asset] + asset.Output = asset.Output.Add(asset.Output, posting.Amount) + + account = m.Accounts[posting.Destination] + asset = account.Volumes[posting.Asset] + asset.Input = asset.Input.Add(asset.Input, posting.Amount) + } + for _, posting := range transaction.Postings { + expandedTransaction.PostCommitVolumes.AddOutput(posting.Source, posting.Asset, + m.Accounts[posting.Source].Volumes[posting.Asset].Output) + expandedTransaction.PostCommitVolumes.AddInput(posting.Source, posting.Asset, + m.Accounts[posting.Source].Volumes[posting.Asset].Input) + + expandedTransaction.PostCommitVolumes.AddOutput(posting.Destination, posting.Asset, + m.Accounts[posting.Destination].Volumes[posting.Asset].Output) + expandedTransaction.PostCommitVolumes.AddInput(posting.Destination, posting.Asset, + m.Accounts[posting.Destination].Volumes[posting.Asset].Input) + } + + m.Transactions = append(m.Transactions, expandedTransaction) + } + return nil +} + +func (m *InMemoryStore) UpdateTransactionsMetadata(ctx context.Context, update ...core.TransactionWithMetadata) error { + for _, tx := range update { + m.Transactions[tx.ID].Metadata = m.Transactions[tx.ID].Metadata.Merge(tx.Metadata) + } + return nil +} + +func (m *InMemoryStore) EnsureAccountsExist(ctx context.Context, accounts []string) error { + for _, address := range accounts { + _, ok := m.Accounts[address] + if ok { + continue + } + m.Accounts[address] = &core.AccountWithVolumes{ + Account: core.Account{ + Address: address, + Metadata: metadata.Metadata{}, + }, + Volumes: core.VolumesByAssets{}, + } + } + return nil +} + +func (m *InMemoryStore) IsInitialized() bool { + return true +} + +func (m *InMemoryStore) GetNextLogID(ctx context.Context) (uint64, error) { + for _, log := range m.Logs { + if !log.Projected { + return log.ID, nil + } + } + return uint64(len(m.Logs)), nil +} + +func (m *InMemoryStore) ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core.ChainedLog, error) { + if idMax > uint64(len(m.Logs)) { + idMax = uint64(len(m.Logs)) + } + + if idMin < uint64(len(m.Logs)) { + return collectionutils.Map(m.Logs[idMin:idMax], func(from *core.ChainedLog) core.ChainedLog { + return *from + }), nil + } + + return []core.ChainedLog{}, nil +} + +func (m *InMemoryStore) GetAccountWithVolumes(ctx context.Context, address string) (*core.AccountWithVolumes, error) { + account, ok := m.Accounts[address] + if !ok { + return &core.AccountWithVolumes{ + Account: core.Account{ + Address: address, + Metadata: metadata.Metadata{}, + }, + Volumes: core.VolumesByAssets{}, + }, nil + } + return account, nil +} + +func (m *InMemoryStore) GetTransaction(ctx context.Context, id uint64) (*core.ExpandedTransaction, error) { + return m.Transactions[id], nil +} + +func NewInMemoryStore() *InMemoryStore { + return &InMemoryStore{ + Accounts: make(map[string]*core.AccountWithVolumes), + } +} diff --git a/components/ledger/pkg/storage/ledgerstore/accounts.go b/components/ledger/pkg/storage/ledgerstore/accounts.go index f1306e03bf..7ce1e55003 100644 --- a/components/ledger/pkg/storage/ledgerstore/accounts.go +++ b/components/ledger/pkg/storage/ledgerstore/accounts.go @@ -4,30 +4,70 @@ import ( "context" "database/sql" "fmt" - "math/big" "regexp" - "strconv" "strings" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/api" "github.com/formancehq/stack/libs/go-libs/metadata" - "github.com/pkg/errors" "github.com/uptrace/bun" - "github.com/uptrace/bun/extra/bunbig" ) const ( accountsTableName = "accounts" ) +type AccountsQuery OffsetPaginatedQuery[AccountsQueryFilters] + +type AccountsQueryFilters struct { + AfterAddress string `json:"after"` + Address string `json:"address"` + Metadata metadata.Metadata `json:"metadata"` +} + +func NewAccountsQuery() AccountsQuery { + return AccountsQuery{ + PageSize: QueryDefaultPageSize, + Order: OrderAsc, + Filters: AccountsQueryFilters{ + Metadata: metadata.Metadata{}, + }, + } +} + +func (a AccountsQuery) WithPageSize(pageSize uint64) AccountsQuery { + if pageSize != 0 { + a.PageSize = pageSize + } + + return a +} + +func (a AccountsQuery) WithAfterAddress(after string) AccountsQuery { + a.Filters.AfterAddress = after + + return a +} + +func (a AccountsQuery) WithAddressFilter(address string) AccountsQuery { + a.Filters.Address = address + + return a +} + +func (a AccountsQuery) WithMetadataFilter(metadata metadata.Metadata) AccountsQuery { + a.Filters.Metadata = metadata + + return a +} + // This regexp is used to validate the account name // If the account name is not valid, it means that the user putted a regex in // the address filter, and we have to change the postgres operator used. var accountNameRegex = regexp.MustCompile(`^[a-zA-Z_0-9]+$`) -type Accounts struct { +type Account struct { bun.BaseModel `bun:"accounts,alias:accounts"` Address string `bun:"address,type:varchar,unique,notnull"` @@ -36,274 +76,109 @@ type Accounts struct { } func (s *Store) buildAccountsQuery(p AccountsQuery) *bun.SelectQuery { - sb := s.schema.NewSelect(accountsTableName). - Model((*Accounts)(nil)). - Column("address", "metadata") + query := s.schema.NewSelect(accountsTableName). + Model((*Account)(nil)). + ColumnExpr("coalesce(accounts.address, moves.account) as address"). + ColumnExpr("coalesce(accounts.metadata, '{}'::jsonb) as metadata"). + Join(fmt.Sprintf(`full join "%s".moves moves on moves.account = accounts.address`, s.schema.Name())) if p.Filters.Address != "" { src := strings.Split(p.Filters.Address, ":") - sb.Where(fmt.Sprintf("jsonb_array_length(address_json) = %d", len(src))) + query.Where(fmt.Sprintf("jsonb_array_length(address_json) = %d", len(src))) for i, segment := range src { - if segment == ".*" || segment == "*" || segment == "" { + if len(segment) == 0 { continue } - - sb.Where(fmt.Sprintf("address_json @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) + query.Where(fmt.Sprintf("address_json @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) } } for key, value := range p.Filters.Metadata { - // TODO: Need to find another way to specify the prefix since Table() methods does not make sense for functions and procedures - sb.Where(s.schema.Table( - fmt.Sprintf("%s(metadata, ?, '%s')", - SQLCustomFuncMetaCompare, strings.ReplaceAll(key, ".", "', '")), - ), value) + query.Where( + fmt.Sprintf(`"%s".%s(metadata, ?, '%s')`, s.schema.Name(), + SQLCustomFuncMetaCompare, strings.ReplaceAll(key, ".", "', '"), + ), value) } - if p.Filters.Balance != "" { - sb.Join("LEFT JOIN " + s.schema.Table(volumesTableName)). - JoinOn("accounts.address = volumes.account") - balanceOperation := "coalesce(volumes.input - volumes.output, 0)" - - balanceValue, err := strconv.ParseInt(p.Filters.Balance, 10, 0) - if err != nil { - // parameter is validated in the controller for now - panic(errors.Wrap(err, "invalid balance parameter")) - } - - if p.Filters.BalanceOperator != "" { - switch p.Filters.BalanceOperator { - case BalanceOperatorLte: - sb.Where(fmt.Sprintf("%s <= ?", balanceOperation), balanceValue) - case BalanceOperatorLt: - sb.Where(fmt.Sprintf("%s < ?", balanceOperation), balanceValue) - case BalanceOperatorGte: - sb.Where(fmt.Sprintf("%s >= ?", balanceOperation), balanceValue) - case BalanceOperatorGt: - sb.Where(fmt.Sprintf("%s > ?", balanceOperation), balanceValue) - case BalanceOperatorE: - sb.Where(fmt.Sprintf("%s = ?", balanceOperation), balanceValue) - case BalanceOperatorNe: - sb.Where(fmt.Sprintf("%s != ?", balanceOperation), balanceValue) - default: - // parameter is validated in the controller for now - panic("invalid balance operator parameter") - } - } else { // if no operator is given, default to gte - sb.Where(fmt.Sprintf("%s >= ?", balanceOperation), balanceValue) - } - } - - return sb + return s.schema.IDB.NewSelect(). + With("cte1", query). + DistinctOn("cte1.address"). + ColumnExpr("cte1.address"). + ColumnExpr("cte1.metadata"). + Table("cte1") } func (s *Store) GetAccounts(ctx context.Context, q AccountsQuery) (*api.Cursor[core.Account], error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_accounts") - defer recordMetrics() - - return UsingOffset(ctx, s.buildAccountsQuery(q), OffsetPaginatedQuery[AccountsQueryFilters](q), - func(account *core.Account, scanner interface{ Scan(args ...any) error }) error { - return scanner.Scan(&account.Address, &account.Metadata) - }) + return UsingOffset[AccountsQueryFilters, core.Account](ctx, + s.buildAccountsQuery(q), OffsetPaginatedQuery[AccountsQueryFilters](q)) } func (s *Store) GetAccount(ctx context.Context, addr string) (*core.Account, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_account") - defer recordMetrics() - - query := s.schema.NewSelect(accountsTableName). - Model((*Accounts)(nil)). - Column("address", "metadata"). + account := &core.Account{} + if err := s.schema.NewSelect(accountsTableName). + ColumnExpr("address"). + ColumnExpr("metadata"). Where("address = ?", addr). - String() - - row := s.schema.QueryRowContext(ctx, query) - if err := row.Err(); err != nil { - return nil, storageerrors.PostgresError(err) - } - - var account core.Account - err := row.Scan(&account.Address, &account.Metadata) - if err != nil { - return nil, storageerrors.PostgresError(err) - } - - return &account, nil -} - -func (s *Store) getAccountWithVolumes(ctx context.Context, exec interface { - QueryContext( - ctx context.Context, query string, args ...interface{}, - ) (*sql.Rows, error) -}, account string) (*core.AccountWithVolumes, error) { - - query := s.schema.NewSelect(accountsTableName). - Model((*Accounts)(nil)). - ColumnExpr("accounts.metadata, volumes.asset, volumes.input, volumes.output"). - Join("LEFT OUTER JOIN "+s.schema.Table(volumesTableName)+" volumes"). - JoinOn("accounts.address = volumes.account"). - Where("accounts.address = ?", account).String() - - rows, err := exec.QueryContext(ctx, query) - if err != nil { - return nil, storageerrors.PostgresError(err) - } - defer rows.Close() - - acc := core.Account{ - Address: account, - Metadata: metadata.Metadata{}, - } - assetsVolumes := core.AssetsVolumes{} - - for rows.Next() { - var asset, inputStr, outputStr sql.NullString - if err := rows.Scan(&acc.Metadata, &asset, &inputStr, &outputStr); err != nil { - return nil, storageerrors.PostgresError(err) - } - - if asset.Valid { - assetsVolumes[asset.String] = core.NewEmptyVolumes() - - if inputStr.Valid { - input, ok := new(big.Int).SetString(inputStr.String, 10) - if !ok { - panic("unable to create big int") - } - assetsVolumes[asset.String] = core.Volumes{ - Input: input, - Output: assetsVolumes[asset.String].Output, - } - } - - if outputStr.Valid { - output, ok := new(big.Int).SetString(outputStr.String, 10) - if !ok { - panic("unable to create big int") - } - assetsVolumes[asset.String] = core.Volumes{ - Input: assetsVolumes[asset.String].Input, - Output: output, - } - } + Model(account). + Scan(ctx, account); err != nil { + if err == sql.ErrNoRows { + return &core.Account{ + Address: addr, + Metadata: metadata.Metadata{}, + }, nil } - } - if err := rows.Err(); err != nil { - return nil, storageerrors.PostgresError(err) + return nil, err } - return &core.AccountWithVolumes{ - Account: acc, - Volumes: assetsVolumes, - }, nil + return account, nil } func (s *Store) GetAccountWithVolumes(ctx context.Context, account string) (*core.AccountWithVolumes, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) + cte2 := s.schema.NewSelect(accountsTableName). + Join(fmt.Sprintf(`full join "%s".moves moves on moves.account = accounts.address`, s.schema.Name())). + Where("account = ?", account). + Group("moves.asset"). + Column("moves.asset"). + ColumnExpr(fmt.Sprintf(`"%s".first(moves.post_commit_input_value order by moves.timestamp desc) as post_commit_input_value`, s.schema.Name())). + ColumnExpr(fmt.Sprintf(`"%s".first(moves.post_commit_output_value order by moves.timestamp desc) as post_commit_output_value`, s.schema.Name())) + + cte3 := s.schema.IDB.NewSelect(). + ColumnExpr(`('{"' || data.asset || '": {"input": ' || data.post_commit_input_value || ', "output": ' || data.post_commit_output_value || '}}')::jsonb as asset`). + TableExpr("cte2 data") + + cte4 := s.schema.IDB.NewSelect(). + ColumnExpr(fmt.Sprintf(`'%s' as account`, account)). + ColumnExpr(fmt.Sprintf(`"%s".aggregate_objects(data.asset) as volumes`, s.schema.Name())). + TableExpr("cte3 data") + + accountWithVolumes := &core.AccountWithVolumes{} + err := s.schema.NewSelect(accountsTableName). + With("cte2", cte2). + With("cte3", cte3). + With("cte4", cte4). + ColumnExpr(fmt.Sprintf("'%s' as address", account)). + ColumnExpr("coalesce(accounts.metadata, '{}'::jsonb) as metadata"). + ColumnExpr("cte4.volumes"). + Join(`right join cte4 on cte4.account = accounts.address`). + Scan(ctx, accountWithVolumes) + if err != nil { + return nil, storageerrors.PostgresError(err) } - recordMetrics := s.instrumentalized(ctx, "get_account_with_volumes") - defer recordMetrics() - return s.getAccountWithVolumes(ctx, s.schema, account) + return accountWithVolumes, nil } func (s *Store) CountAccounts(ctx context.Context, q AccountsQuery) (uint64, error) { - if !s.isInitialized { - return 0, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "count_accounts") - defer recordMetrics() - sb := s.buildAccountsQuery(q) count, err := sb.Count(ctx) return uint64(count), storageerrors.PostgresError(err) } -func (s *Store) EnsureAccountExists(ctx context.Context, account string) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "ensure_account_exists") - defer recordMetrics() - - a := &Accounts{ - Address: account, - Metadata: metadata.Metadata{}, - AddressJson: strings.Split(account, ":"), - } - - _, err := s.schema.NewInsert(accountsTableName). - Model(a). - Ignore(). - Exec(ctx) - - return storageerrors.PostgresError(err) -} - -func (s *Store) EnsureAccountsExist(ctx context.Context, accounts []string) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "ensure_accounts_exist") - defer recordMetrics() - - accs := make([]*Accounts, len(accounts)) +func (s *Store) UpdateAccountsMetadata(ctx context.Context, accounts ...core.Account) error { + accs := make([]*Account, len(accounts)) for i, a := range accounts { - accs[i] = &Accounts{ - Address: a, - Metadata: metadata.Metadata{}, - AddressJson: strings.Split(a, ":"), - } - } - - _, err := s.schema.NewInsert(accountsTableName). - Model(&accs). - Ignore(). - Exec(ctx) - - return storageerrors.PostgresError(err) -} - -func (s *Store) UpdateAccountMetadata(ctx context.Context, address string, metadata metadata.Metadata) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "update_account_metadata") - defer recordMetrics() - - a := &Accounts{ - Address: address, - Metadata: metadata, - AddressJson: strings.Split(address, ":"), - } - - _, err := s.schema.NewInsert(accountsTableName). - Model(a). - On("CONFLICT (address) DO UPDATE"). - Set("metadata = accounts.metadata || EXCLUDED.metadata"). - Exec(ctx) - - return storageerrors.PostgresError(err) -} - -func (s *Store) UpdateAccountsMetadata(ctx context.Context, accounts []core.Account) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "update_accounts_metadata") - defer recordMetrics() - - accs := make([]*Accounts, len(accounts)) - for i, a := range accounts { - accs[i] = &Accounts{ + accs[i] = &Account{ Address: a.Address, Metadata: a.Metadata, AddressJson: strings.Split(a.Address, ":"), @@ -318,120 +193,3 @@ func (s *Store) UpdateAccountsMetadata(ctx context.Context, accounts []core.Acco return storageerrors.PostgresError(err) } - -func (s *Store) GetBalanceFromLogs(ctx context.Context, address, asset string) (*big.Int, error) { - selectLogsForExistingAccount := s.schema. - NewSelect(LogTableName). - Model(&LogsV2{}). - Where(fmt.Sprintf(`data->'transaction'->'postings' @> '[{"destination": "%s", "asset": "%s"}]' OR data->'transaction'->'postings' @> '[{"source": "%s", "asset": "%s"}]'`, address, asset, address, asset)) - - selectPostings := s.schema.IDB.NewSelect(). - TableExpr(`(` + selectLogsForExistingAccount.String() + `) as logs`). - ColumnExpr("jsonb_array_elements(logs.data->'transaction'->'postings') as postings") - - selectBalances := s.schema.IDB.NewSelect(). - TableExpr(`(` + selectPostings.String() + `) as postings`). - ColumnExpr(fmt.Sprintf("SUM(CASE WHEN (postings.postings::jsonb)->>'source' = '%s' THEN -((((postings.postings::jsonb)->'amount')::numeric)) ELSE ((postings.postings::jsonb)->'amount')::numeric END)", address)) - - row := s.schema.IDB.QueryRowContext(ctx, selectBalances.String()) - if row.Err() != nil { - return nil, row.Err() - } - - var balance *bunbig.Int - if err := row.Scan(&balance); err != nil { - return nil, err - } - return (*big.Int)(balance), nil -} - -func (s *Store) GetMetadataFromLogs(ctx context.Context, address, key string) (string, error) { - l := LogsV2{} - if err := s.schema.NewSelect(LogTableName). - Model(&l). - Order("id DESC"). - WhereOr( - "type = ? AND data->>'targetId' = ? AND data->>'targetType' = ? AND "+fmt.Sprintf("data->'metadata' ? '%s'", key), - core.SetMetadataLogType, address, core.MetaTargetTypeAccount, - ). - WhereOr( - "type = ? AND "+fmt.Sprintf("data->'accountMetadata'->'%s' ? '%s'", address, key), - core.NewTransactionLogType, - ). - Limit(1). - Scan(ctx); err != nil { - return "", storageerrors.PostgresError(err) - } - - payload, err := core.HydrateLog(core.LogType(l.Type), l.Data) - if err != nil { - panic(errors.Wrap(err, "hydrating log data")) - } - - switch payload := payload.(type) { - case core.NewTransactionLogPayload: - return payload.AccountMetadata[address][key], nil - case core.SetMetadataLogPayload: - return payload.Metadata[key], nil - default: - panic("should not happen") - } -} - -type AccountsQuery OffsetPaginatedQuery[AccountsQueryFilters] - -type AccountsQueryFilters struct { - AfterAddress string `json:"after"` - Address string `json:"address"` - Balance string `json:"balance"` - BalanceOperator BalanceOperator `json:"balanceOperator"` - Metadata metadata.Metadata `json:"metadata"` -} - -func NewAccountsQuery() AccountsQuery { - return AccountsQuery{ - PageSize: QueryDefaultPageSize, - Order: OrderAsc, - Filters: AccountsQueryFilters{ - Metadata: metadata.Metadata{}, - }, - } -} - -func (a AccountsQuery) WithPageSize(pageSize uint64) AccountsQuery { - if pageSize != 0 { - a.PageSize = pageSize - } - - return a -} - -func (a AccountsQuery) WithAfterAddress(after string) AccountsQuery { - a.Filters.AfterAddress = after - - return a -} - -func (a AccountsQuery) WithAddressFilter(address string) AccountsQuery { - a.Filters.Address = address - - return a -} - -func (a AccountsQuery) WithBalanceFilter(balance string) AccountsQuery { - a.Filters.Balance = balance - - return a -} - -func (a AccountsQuery) WithBalanceOperatorFilter(balanceOperator BalanceOperator) AccountsQuery { - a.Filters.BalanceOperator = balanceOperator - - return a -} - -func (a AccountsQuery) WithMetadataFilter(metadata metadata.Metadata) AccountsQuery { - a.Filters.Metadata = metadata - - return a -} diff --git a/components/ledger/pkg/storage/ledgerstore/accounts_test.go b/components/ledger/pkg/storage/ledgerstore/accounts_test.go index d95105ab2a..53ead9c4a0 100644 --- a/components/ledger/pkg/storage/ledgerstore/accounts_test.go +++ b/components/ledger/pkg/storage/ledgerstore/accounts_test.go @@ -2,160 +2,38 @@ package ledgerstore_test import ( "context" - "fmt" "math/big" "testing" "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/metadata" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestGetBalanceFromLogs(t *testing.T) { +func TestUpdateAccountsMetadata(t *testing.T) { t.Parallel() store := newLedgerStore(t) - const batchNumber = 100 - const batchSize = 10 - const input = 100 - const output = 10 - - logs := make([]*core.ActiveLog, 0) - for i := 0; i < batchNumber; i++ { - for j := 0; j < batchSize; j++ { - activeLog := core.NewActiveLog(core.NewTransactionLog( - core.NewTransaction().WithPostings( - core.NewPosting("world", fmt.Sprintf("account:%d", j), "EUR/2", big.NewInt(input)), - core.NewPosting(fmt.Sprintf("account:%d", j), "starbucks", "EUR/2", big.NewInt(output)), - ).WithID(uint64(i*batchSize+j)), - map[string]metadata.Metadata{}, - )) - logs = append(logs, activeLog) - } - } - _, err := store.InsertLogs(context.Background(), logs) - require.NoError(t, err) - - balance, err := store.GetBalanceFromLogs(context.Background(), "account:1", "EUR/2") - require.NoError(t, err) - require.Equal(t, big.NewInt((input-output)*batchNumber), balance) -} - -func TestGetMetadataFromLogs(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - logs := make([]*core.ActiveLog, 0) - logs = append(logs, core.NewActiveLog(core.NewTransactionLog( - core.NewTransaction().WithPostings( - core.NewPosting("world", "bank", "EUR/2", big.NewInt(100)), - core.NewPosting("bank", "starbucks", "EUR/2", big.NewInt(10)), - ), - map[string]metadata.Metadata{}, - ))) - logs = append(logs, core.NewActiveLog(core.NewSetMetadataLog(core.Now(), core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeAccount, - TargetID: "bank", - Metadata: metadata.Metadata{ - "foo": "bar", - }, - }))) - logs = append(logs, core.NewActiveLog(core.NewTransactionLog( - core.NewTransaction().WithPostings( - core.NewPosting("world", "bank", "EUR/2", big.NewInt(100)), - core.NewPosting("bank", "starbucks", "EUR/2", big.NewInt(10)), - ).WithID(1), - map[string]metadata.Metadata{}, - ))) - logs = append(logs, core.NewActiveLog(core.NewSetMetadataLog(core.Now(), core.SetMetadataLogPayload{ - TargetType: core.MetaTargetTypeAccount, - TargetID: "bank", - Metadata: metadata.Metadata{ - "role": "admin", - }, - }))) - logs = append(logs, core.NewActiveLog(core.NewTransactionLog( - core.NewTransaction().WithPostings( - core.NewPosting("world", "bank", "EUR/2", big.NewInt(100)), - core.NewPosting("bank", "starbucks", "EUR/2", big.NewInt(10)), - ).WithID(2), - map[string]metadata.Metadata{}, - ))) - - _, err := store.InsertLogs(context.Background(), logs) - require.NoError(t, err) - - metadata, err := store.GetMetadataFromLogs(context.Background(), "bank", "foo") - require.NoError(t, err) - require.Equal(t, "bar", metadata) -} - -func TestAccounts(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - t.Run("success balance", func(t *testing.T) { - q := ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithBalanceFilter("50") - - _, err := store.GetAccounts(context.Background(), q) - assert.NoError(t, err, "balance filter should not fail") - }) - - t.Run("panic invalid balance", func(t *testing.T) { - q := ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithBalanceFilter("TEST") - - assert.PanicsWithError( - t, `invalid balance parameter: strconv.ParseInt: parsing "TEST": invalid syntax`, - - func() { - _, _ = store.GetAccounts(context.Background(), q) - }, "invalid balance in storage should panic") - }) - - t.Run("panic invalid balance operator", func(t *testing.T) { - assert.PanicsWithValue(t, "invalid balance operator parameter", func() { - q := ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithBalanceFilter("50"). - WithBalanceOperatorFilter("TEST") - - _, _ = store.GetAccounts(context.Background(), q) - }, "invalid balance operator in storage should panic") - }) - - t.Run("success balance operator", func(t *testing.T) { - q := ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithBalanceFilter("50"). - WithBalanceOperatorFilter(ledgerstore.BalanceOperatorLte) - - _, err := store.GetAccounts(context.Background(), q) - assert.NoError(t, err, "balance operator filter should not fail") - }) - - t.Run("success account insertion", func(t *testing.T) { - addr := "test:account" + t.Run("update metadata", func(t *testing.T) { metadata := metadata.Metadata{ "foo": "bar", } - err := store.UpdateAccountMetadata(context.Background(), addr, metadata) - assert.NoError(t, err, "account insertion should not fail") + err := store.UpdateAccountsMetadata(context.Background(), core.Account{ + Address: "bank", + Metadata: metadata, + }) + require.NoError(t, err, "account insertion should not fail") - account, err := store.GetAccount(context.Background(), addr) - assert.NoError(t, err, "account retrieval should not fail") + account, err := store.GetAccount(context.Background(), "bank") + require.NoError(t, err, "account retrieval should not fail") - assert.Equal(t, addr, account.Address, "account address should match") - assert.Equal(t, metadata, account.Metadata, "account metadata should match") + require.Equal(t, "bank", account.Address, "account address should match") + require.Equal(t, metadata, account.Metadata, "account metadata should match") }) - t.Run("success multiple account insertions", func(t *testing.T) { + t.Run("success updating multiple account metadata", func(t *testing.T) { accounts := []core.Account{ { Address: "test:account1", @@ -171,41 +49,97 @@ func TestAccounts(t *testing.T) { }, } - err := store.UpdateAccountsMetadata(context.Background(), accounts) - assert.NoError(t, err, "account insertion should not fail") + err := store.UpdateAccountsMetadata(context.Background(), accounts...) + require.NoError(t, err, "account insertion should not fail") for _, account := range accounts { acc, err := store.GetAccount(context.Background(), account.Address) - assert.NoError(t, err, "account retrieval should not fail") + require.NoError(t, err, "account retrieval should not fail") - assert.Equal(t, account.Address, acc.Address, "account address should match") - assert.Equal(t, account.Metadata, acc.Metadata, "account metadata should match") + require.Equal(t, account.Address, acc.Address, "account address should match") + require.Equal(t, account.Metadata, acc.Metadata, "account metadata should match") } }) +} - t.Run("success ensure account exists", func(t *testing.T) { - addr := "test:account:4" +func TestGetAccount(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) - err := store.EnsureAccountExists(context.Background(), addr) - assert.NoError(t, err, "account insertion should not fail") + require.NoError(t, insertTransactions(context.Background(), store, + *core.NewTransaction().WithPostings( + core.NewPosting("world", "multi", "USD/2", big.NewInt(100)), + ), + )) - account, err := store.GetAccount(context.Background(), addr) - assert.NoError(t, err, "account retrieval should not fail") + account, err := store.GetAccount(context.Background(), "multi") + require.NoError(t, err) + require.Equal(t, core.Account{ + Address: "multi", + Metadata: metadata.Metadata{}, + }, *account) +} - assert.Equal(t, addr, account.Address, "account address should match") - }) +func TestGetAccountWithVolumes(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) - t.Run("success ensure multiple accounts exist", func(t *testing.T) { - addrs := []string{"test:account:4", "test:account:5", "test:account:6"} + require.NoError(t, insertTransactions(context.Background(), store, + *core.NewTransaction().WithPostings( + core.NewPosting("world", "multi", "USD/2", big.NewInt(100)), + ), + )) - err := store.EnsureAccountsExist(context.Background(), addrs) - assert.NoError(t, err, "account insertion should not fail") + accountWithVolumes, err := store.GetAccountWithVolumes(context.Background(), "multi") + require.NoError(t, err) + require.Equal(t, &core.AccountWithVolumes{ + Account: core.Account{ + Address: "multi", + Metadata: metadata.Metadata{}, + }, + Volumes: map[string]*core.Volumes{ + "USD/2": core.NewEmptyVolumes().WithInputInt64(100), + }, + }, accountWithVolumes) +} - for _, addr := range addrs { - account, err := store.GetAccount(context.Background(), addr) - assert.NoError(t, err, "account retrieval should not fail") +func TestUpdateAccountMetadata(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) - assert.Equal(t, addr, account.Address, "account address should match") - } + err := store.UpdateAccountsMetadata(context.Background(), core.Account{ + Address: "central_bank", + Metadata: metadata.Metadata{ + "foo": "bar", + }, }) + require.NoError(t, err) + + account, err := store.GetAccount(context.Background(), "central_bank") + require.NoError(t, err) + require.EqualValues(t, "bar", account.Metadata["foo"]) +} + +func TestGetAccountWithAccountNotExisting(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + + account, err := store.GetAccount(context.Background(), "account_not_existing") + require.NoError(t, err) + require.NotNil(t, account) +} + +func TestCountAccounts(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + + require.NoError(t, insertTransactions(context.Background(), store, + *core.NewTransaction().WithPostings( + core.NewPosting("world", "central_bank", "USD/2", big.NewInt(100)), + ), + )) + + countAccounts, err := store.CountAccounts(context.Background(), ledgerstore.AccountsQuery{}) + require.NoError(t, err) + require.EqualValues(t, 2, countAccounts) // world + central_bank } diff --git a/components/ledger/pkg/storage/ledgerstore/balances.go b/components/ledger/pkg/storage/ledgerstore/balances.go index 61ea7fc272..722b06219c 100644 --- a/components/ledger/pkg/storage/ledgerstore/balances.go +++ b/components/ledger/pkg/storage/ledgerstore/balances.go @@ -2,204 +2,163 @@ package ledgerstore import ( "context" + "database/sql" + "encoding/json" "fmt" "math/big" - "strconv" "strings" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/api" - "github.com/lib/pq" ) -func (s *Store) GetBalancesAggregated(ctx context.Context, q BalancesQuery) (core.AssetsBalances, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_balances_aggregated") - defer recordMetrics() +type BalancesQueryFilters struct { + AfterAddress string `json:"afterAddress"` + AddressRegexp string `json:"addressRegexp"` +} - sb := s.schema.NewSelect(volumesTableName). - Model((*Volumes)(nil)). - ColumnExpr("asset"). - ColumnExpr("sum(input - output) as arr"). - Group("asset") +type BalancesQuery OffsetPaginatedQuery[BalancesQueryFilters] - if q.Filters.AddressRegexp != "" { - src := strings.Split(q.Filters.AddressRegexp, ":") - sb.Where(fmt.Sprintf("jsonb_array_length(account_json) = %d", len(src))) +func NewBalancesQuery() BalancesQuery { + return BalancesQuery{ + PageSize: QueryDefaultPageSize, + Order: OrderAsc, + Filters: BalancesQueryFilters{}, + } +} - for i, segment := range src { - if segment == ".*" || segment == "*" || segment == "" { - continue - } +func (q BalancesQuery) GetPageSize() uint64 { + return q.PageSize +} - sb.Where(fmt.Sprintf("account_json @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) - } - } +func (b BalancesQuery) WithAfterAddress(after string) BalancesQuery { + b.Filters.AfterAddress = after - rows, err := s.schema.QueryContext(ctx, sb.String()) - if err != nil { - return nil, storageerrors.PostgresError(err) - } - defer rows.Close() + return b +} - aggregatedBalances := core.AssetsBalances{} +func (b BalancesQuery) WithAddressFilter(address string) BalancesQuery { + b.Filters.AddressRegexp = address - for rows.Next() { - var ( - asset string - balancesStr string - ) - if err = rows.Scan(&asset, &balancesStr); err != nil { - return nil, storageerrors.PostgresError(err) - } + return b +} - balances, ok := new(big.Int).SetString(balancesStr, 10) - if !ok { - panic("unable to restore big int") - } +func (b BalancesQuery) WithPageSize(pageSize uint64) BalancesQuery { + b.PageSize = pageSize + return b +} + +type balancesByAssets core.BalancesByAssets - aggregatedBalances[asset] = balances +func (b *balancesByAssets) Scan(value interface{}) error { + var i sql.NullString + if err := i.Scan(value); err != nil { + return err } - if err := rows.Err(); err != nil { - return nil, storageerrors.PostgresError(err) + + *b = balancesByAssets{} + if err := json.Unmarshal([]byte(i.String), b); err != nil { + return err } - return aggregatedBalances, nil + return nil } -func (s *Store) GetBalances(ctx context.Context, q BalancesQuery) (*api.Cursor[core.AccountsBalances], error) { - if !s.isInitialized { - return nil, - storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_balances") - defer recordMetrics() - - sb := s.schema.NewSelect(volumesTableName). - Model((*Volumes)(nil)). +func (s *Store) GetBalancesAggregated(ctx context.Context, q BalancesQuery) (core.BalancesByAssets, error) { + selectLastMoveForEachAccountAsset := s.schema.NewSelect(MovesTableName). ColumnExpr("account"). - ColumnExpr("array_agg((asset, input - output)) as arr"). - Group("account", "account_json"). - Order("account DESC") - - if q.Filters.AfterAddress != "" { - sb.Where("account < ?", q.Filters.AfterAddress) - } + ColumnExpr("asset"). + ColumnExpr(fmt.Sprintf(`"%s".first(post_commit_input_value order by timestamp desc) as post_commit_input_value`, s.schema.Name())). + ColumnExpr(fmt.Sprintf(`"%s".first(post_commit_output_value order by timestamp desc) as post_commit_output_value`, s.schema.Name())). + GroupExpr("account, asset") if q.Filters.AddressRegexp != "" { src := strings.Split(q.Filters.AddressRegexp, ":") - sb.Where(fmt.Sprintf("jsonb_array_length(account_json) = %d", len(src))) + selectLastMoveForEachAccountAsset.Where(fmt.Sprintf("jsonb_array_length(account_array) = %d", len(src))) for i, segment := range src { - if segment == ".*" || segment == "*" || segment == "" { + if segment == "" { continue } - - sb.Where(fmt.Sprintf("account_json @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) + selectLastMoveForEachAccountAsset.Where(fmt.Sprintf("account_array @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) } } - return UsingOffset(ctx, sb, OffsetPaginatedQuery[BalancesQueryFilters](q), - func(accountsBalances *core.AccountsBalances, scanner interface{ Scan(args ...any) error }) error { - var currentAccount string - var arrayAgg []string - if err := scanner.Scan(¤tAccount, pq.Array(&arrayAgg)); err != nil { - return err - } - - *accountsBalances = core.AccountsBalances{ - currentAccount: map[string]*big.Int{}, - } - - // arrayAgg is in the form: []string{"(USD,-250)","(EUR,1000)"} - for _, agg := range arrayAgg { - // Remove parenthesis - agg = agg[1 : len(agg)-1] - // Split the asset and balances on the comma separator - split := strings.Split(agg, ",") - asset := split[0] - balancesString := split[1] - balances, err := strconv.ParseInt(balancesString, 10, 64) - if err != nil { - return err - } - (*accountsBalances)[currentAccount][asset] = big.NewInt(balances) - } - - return nil - }) -} - -type BalanceOperator string - -const ( - BalanceOperatorE BalanceOperator = "e" - BalanceOperatorGt BalanceOperator = "gt" - BalanceOperatorGte BalanceOperator = "gte" - BalanceOperatorLt BalanceOperator = "lt" - BalanceOperatorLte BalanceOperator = "lte" - BalanceOperatorNe BalanceOperator = "ne" - - DefaultBalanceOperator = BalanceOperatorGte -) - -func (b BalanceOperator) IsValid() bool { - switch b { - case BalanceOperatorE, - BalanceOperatorGt, - BalanceOperatorGte, - BalanceOperatorLt, - BalanceOperatorNe, - BalanceOperatorLte: - return true + type row struct { + Asset string `bun:"asset"` + Aggregated *Int `bun:"aggregated"` } - return false -} + rows := make([]row, 0) + if err := s.schema.IDB.NewSelect(). + With("cte1", selectLastMoveForEachAccountAsset). + Column("asset"). + ColumnExpr("sum(cte1.post_commit_input_value) - sum(cte1.post_commit_output_value) as aggregated"). + Table("cte1"). + Group("cte1.asset"). + Scan(ctx, &rows); err != nil { + return nil, storageerrors.PostgresError(err) + } -func NewBalanceOperator(s string) (BalanceOperator, bool) { - if !BalanceOperator(s).IsValid() { - return "", false + aggregatedBalances := core.BalancesByAssets{} + for _, row := range rows { + aggregatedBalances[row.Asset] = (*big.Int)(row.Aggregated) } - return BalanceOperator(s), true + return aggregatedBalances, nil } -type BalancesQueryFilters struct { - AfterAddress string `json:"afterAddress"` - AddressRegexp string `json:"addressRegexp"` -} +func (s *Store) GetBalances(ctx context.Context, q BalancesQuery) (*api.Cursor[core.BalancesByAssetsByAccounts], error) { + selectLastMoveForEachAccountAsset := s.schema.NewSelect(MovesTableName). + ColumnExpr("account"). + ColumnExpr("asset"). + ColumnExpr(fmt.Sprintf(`"%s".first(post_commit_input_value order by timestamp desc) as post_commit_input_value`, s.schema.Name())). + ColumnExpr(fmt.Sprintf(`"%s".first(post_commit_output_value order by timestamp desc) as post_commit_output_value`, s.schema.Name())). + GroupExpr("account, asset") -type BalancesQuery OffsetPaginatedQuery[BalancesQueryFilters] + if q.Filters.AddressRegexp != "" { + src := strings.Split(q.Filters.AddressRegexp, ":") + selectLastMoveForEachAccountAsset.Where(fmt.Sprintf("jsonb_array_length(account_array) = %d", len(src))) -func NewBalancesQuery() BalancesQuery { - return BalancesQuery{ - PageSize: QueryDefaultPageSize, - Order: OrderAsc, - Filters: BalancesQueryFilters{}, + for i, segment := range src { + if len(segment) == 0 { + continue + } + selectLastMoveForEachAccountAsset.Where(fmt.Sprintf("account_array @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) + } } -} - -func (q BalancesQuery) GetPageSize() uint64 { - return q.PageSize -} -func (b BalancesQuery) WithAfterAddress(after string) BalancesQuery { - b.Filters.AfterAddress = after - - return b -} + if q.Filters.AfterAddress != "" { + selectLastMoveForEachAccountAsset.Where("account > ?", q.Filters.AfterAddress) + } -func (b BalancesQuery) WithAddressFilter(address string) BalancesQuery { - b.Filters.AddressRegexp = address + query := s.schema.IDB.NewSelect(). + With("cte1", selectLastMoveForEachAccountAsset). + Column("data.account"). + ColumnExpr(fmt.Sprintf(`"%s".aggregate_objects(data.asset) as balances_by_assets`, s.schema.Name())). + TableExpr(`( + select data.account, ('{"' || data.asset || '": ' || sum(data.post_commit_input_value) - sum(data.post_commit_output_value) || '}')::jsonb as asset + from cte1 data + group by data.account, data.asset + ) data`). + Order("data.account"). + Group("data.account") + + type result struct { + Account string `bun:"account"` + Assets balancesByAssets `bun:"balances_by_assets"` + } - return b -} + cursor, err := UsingOffset[BalancesQueryFilters, result](ctx, + query, OffsetPaginatedQuery[BalancesQueryFilters](q)) + if err != nil { + return nil, err + } -func (b BalancesQuery) WithPageSize(pageSize uint64) BalancesQuery { - b.PageSize = pageSize - return b + return api.MapCursor(cursor, func(from result) core.BalancesByAssetsByAccounts { + return core.BalancesByAssetsByAccounts{ + from.Account: core.BalancesByAssets(from.Assets), + } + }), nil } diff --git a/components/ledger/pkg/storage/ledgerstore/balances_test.go b/components/ledger/pkg/storage/ledgerstore/balances_test.go index 57f8102c2a..a3e1254cda 100644 --- a/components/ledger/pkg/storage/ledgerstore/balances_test.go +++ b/components/ledger/pkg/storage/ledgerstore/balances_test.go @@ -14,17 +14,11 @@ func TestGetBalances(t *testing.T) { t.Parallel() store := newLedgerStore(t) - require.NoError(t, store.UpdateVolumes(context.Background(), core.AccountsAssetsVolumes{ - "world": { - "USD": core.NewEmptyVolumes().WithOutput(big.NewInt(200)), - }, - "users:1": { - "USD": core.NewEmptyVolumes().WithInput(big.NewInt(1)), - }, - "central_bank": { - "USD": core.NewEmptyVolumes().WithInput(big.NewInt(199)), - }, - })) + tx := core.NewTransaction().WithPostings( + core.NewPosting("world", "users:1", "USD", big.NewInt(1)), + core.NewPosting("world", "central_bank", "USD", big.NewInt(199)), + ) + require.NoError(t, insertTransactions(context.Background(), store, *tx)) t.Run("all accounts", func(t *testing.T) { cursor, err := store.GetBalances(context.Background(), @@ -34,20 +28,20 @@ func TestGetBalances(t *testing.T) { require.Equal(t, false, cursor.HasMore) require.Equal(t, "", cursor.Previous) require.Equal(t, "", cursor.Next) - require.Equal(t, []core.AccountsBalances{ + require.Equal(t, []core.BalancesByAssetsByAccounts{ { - "world": core.AssetsBalances{ - "USD": big.NewInt(-200), + "central_bank": core.BalancesByAssets{ + "USD": big.NewInt(199), }, }, { - "users:1": core.AssetsBalances{ + "users:1": core.BalancesByAssets{ "USD": big.NewInt(1), }, }, { - "central_bank": core.AssetsBalances{ - "USD": big.NewInt(199), + "world": core.BalancesByAssets{ + "USD": big.NewInt(-200), }, }, }, cursor.Data) @@ -62,10 +56,10 @@ func TestGetBalances(t *testing.T) { require.Equal(t, true, cursor.HasMore) require.Equal(t, "", cursor.Previous) require.NotEqual(t, "", cursor.Next) - require.Equal(t, []core.AccountsBalances{ + require.Equal(t, []core.BalancesByAssetsByAccounts{ { - "world": core.AssetsBalances{ - "USD": big.NewInt(-200), + "central_bank": core.BalancesByAssets{ + "USD": big.NewInt(199), }, }, }, cursor.Data) @@ -73,22 +67,17 @@ func TestGetBalances(t *testing.T) { t.Run("after", func(t *testing.T) { cursor, err := store.GetBalances(context.Background(), - ledgerstore.NewBalancesQuery().WithPageSize(10).WithAfterAddress("world"), + ledgerstore.NewBalancesQuery().WithPageSize(10).WithAfterAddress("users:1"), ) require.NoError(t, err) require.Equal(t, 10, cursor.PageSize) require.Equal(t, false, cursor.HasMore) require.Equal(t, "", cursor.Previous) require.Equal(t, "", cursor.Next) - require.Equal(t, []core.AccountsBalances{ - { - "users:1": core.AssetsBalances{ - "USD": big.NewInt(1), - }, - }, + require.Equal(t, []core.BalancesByAssetsByAccounts{ { - "central_bank": core.AssetsBalances{ - "USD": big.NewInt(199), + "world": core.BalancesByAssets{ + "USD": big.NewInt(-200), }, }, }, cursor.Data) @@ -98,7 +87,7 @@ func TestGetBalances(t *testing.T) { cursor, err := store.GetBalances(context.Background(), ledgerstore.NewBalancesQuery(). WithPageSize(10). - WithAfterAddress("world"). + WithAfterAddress("central_bank"). WithAddressFilter("users:1"), ) require.NoError(t, err) @@ -106,9 +95,9 @@ func TestGetBalances(t *testing.T) { require.Equal(t, false, cursor.HasMore) require.Equal(t, "", cursor.Previous) require.Equal(t, "", cursor.Next) - require.Equal(t, []core.AccountsBalances{ + require.Equal(t, []core.BalancesByAssetsByAccounts{ { - "users:1": core.AssetsBalances{ + "users:1": core.BalancesByAssets{ "USD": big.NewInt(1), }, }, @@ -120,22 +109,22 @@ func TestGetBalancesAggregated(t *testing.T) { t.Parallel() store := newLedgerStore(t) - require.NoError(t, store.UpdateVolumes(context.Background(), core.AccountsAssetsVolumes{ - "world": { - "USD": core.NewEmptyVolumes().WithOutput(big.NewInt(200)), - }, - "users:1": { - "USD": core.NewEmptyVolumes().WithInput(big.NewInt(1)), - }, - "central_bank": { - "USD": core.NewEmptyVolumes().WithInput(big.NewInt(199)), - }, - })) + tx := core.NewTransaction().WithPostings( + core.NewPosting("world", "users:1", "USD", big.NewInt(1)), + core.NewPosting("world", "users:2", "USD", big.NewInt(199)), + ) + require.NoError(t, insertTransactions(context.Background(), store, *tx)) q := ledgerstore.NewBalancesQuery().WithPageSize(10) cursor, err := store.GetBalancesAggregated(context.Background(), q) require.NoError(t, err) - require.Equal(t, core.AssetsBalances{ + require.Equal(t, core.BalancesByAssets{ "USD": big.NewInt(0), }, cursor) + + ret, err := store.GetBalancesAggregated(context.Background(), ledgerstore.NewBalancesQuery().WithPageSize(10).WithAddressFilter("users:")) + require.NoError(t, err) + require.Equal(t, core.BalancesByAssets{ + "USD": big.NewInt(200), + }, ret) } diff --git a/components/ledger/pkg/storage/ledgerstore/bigint.go b/components/ledger/pkg/storage/ledgerstore/bigint.go new file mode 100644 index 0000000000..5b3cdcf840 --- /dev/null +++ b/components/ledger/pkg/storage/ledgerstore/bigint.go @@ -0,0 +1,88 @@ +package ledgerstore + +import ( + "database/sql" + "database/sql/driver" + "encoding/json" + "fmt" + "math/big" +) + +type Int big.Int + +func (i *Int) MarshalJSON() ([]byte, error) { + return json.Marshal(i.ToMathBig()) +} + +func (i *Int) UnmarshalJSON(bytes []byte) error { + v, err := i.FromString(string(bytes)) + if err != nil { + return err + } + *i = *v + return nil +} + +func NewInt() *Int { + return new(Int) +} +func newBigint(x *big.Int) *Int { + return (*Int)(x) +} + +// same as NewBigint() +func FromMathBig(x *big.Int) *Int { + return (*Int)(x) +} + +func FromInt64(x int64) *Int { + return FromMathBig(big.NewInt(x)) +} + +func (i *Int) FromString(x string) (*Int, error) { + if x == "" { + return FromInt64(0), nil + } + a := big.NewInt(0) + b, ok := a.SetString(x, 10) + + if !ok { + return nil, fmt.Errorf("cannot create Int from string") + } + + return newBigint(b), nil +} + +func (b *Int) Value() (driver.Value, error) { + return (*big.Int)(b).String(), nil +} + +func (b *Int) Set(v *Int) *Int { + return (*Int)((*big.Int)(b).Set((*big.Int)(v))) +} + +func (b *Int) Sub(x *Int, y *Int) *Int { + return (*Int)((*big.Int)(b).Sub((*big.Int)(x), (*big.Int)(y))) +} + +func (b *Int) Scan(value interface{}) error { + + var i sql.NullString + + if err := i.Scan(value); err != nil { + return err + } + + if _, ok := (*big.Int)(b).SetString(i.String, 10); ok { + return nil + } + + return fmt.Errorf("Error converting type %T into Bigint", value) +} + +func (b *Int) ToMathBig() *big.Int { + return (*big.Int)(b) +} + +var _ json.Unmarshaler = (*Int)(nil) +var _ json.Marshaler = (*Int)(nil) diff --git a/components/ledger/pkg/storage/ledgerstore/logs.go b/components/ledger/pkg/storage/ledgerstore/logs.go index 60de9d2e57..99ac12f015 100644 --- a/components/ledger/pkg/storage/ledgerstore/logs.go +++ b/components/ledger/pkg/storage/ledgerstore/logs.go @@ -9,20 +9,65 @@ import ( "math/big" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/api" "github.com/formancehq/stack/libs/go-libs/collectionutils" "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/lib/pq" "github.com/pkg/errors" "github.com/uptrace/bun" + "github.com/uptrace/bun/extra/bunbig" ) const ( - LogTableName = "logs_v2" - LogIngestionTableName = "logs_ingestion" + LogTableName = "logs_v2" ) +type LogsQueryFilters struct { + EndTime core.Time `json:"endTime"` + StartTime core.Time `json:"startTime"` +} + +type LogsQuery ColumnPaginatedQuery[LogsQueryFilters] + +func NewLogsQuery() LogsQuery { + return LogsQuery{ + PageSize: QueryDefaultPageSize, + Column: "id", + Order: OrderDesc, + Filters: LogsQueryFilters{}, + } +} + +func (a LogsQuery) WithPaginationID(id uint64) LogsQuery { + a.PaginationID = &id + return a +} + +func (l LogsQuery) WithPageSize(pageSize uint64) LogsQuery { + if pageSize != 0 { + l.PageSize = pageSize + } + + return l +} + +func (l LogsQuery) WithStartTimeFilter(start core.Time) LogsQuery { + if !start.IsZero() { + l.Filters.StartTime = start + } + + return l +} + +func (l LogsQuery) WithEndTimeFilter(end core.Time) LogsQuery { + if !end.IsZero() { + l.Filters.EndTime = end + } + + return l +} + type AccountWithBalances struct { bun.BaseModel `bun:"accounts,alias:accounts"` @@ -40,33 +85,28 @@ type LogsV2 struct { Date core.Time `bun:"date,type:timestamptz"` Data []byte `bun:"data,type:jsonb"` IdempotencyKey string `bun:"idempotency_key,type:varchar(256),unique"` + Projected bool `bun:"projected,type:boolean"` } -func (log LogsV2) toCore() core.PersistedLog { +func (log LogsV2) toCore() core.ChainedLog { payload, err := core.HydrateLog(core.LogType(log.Type), log.Data) if err != nil { panic(errors.Wrap(err, "hydrating log data")) } - return core.PersistedLog{ + return core.ChainedLog{ Log: core.Log{ Type: core.LogType(log.Type), Data: payload, Date: log.Date.UTC(), IdempotencyKey: log.IdempotencyKey, }, - ID: log.ID, - Hash: log.Hash, + ID: log.ID, + Hash: log.Hash, + Projected: log.Projected, } } -type LogsIngestion struct { - bun.BaseModel `bun:"logs_ingestion,alias:logs_ingestion"` - - OnerowId bool `bun:"onerow_id,pk,type:bool,default:true"` - LogId uint64 `bun:"log_id,type:bigint"` -} - type RawMessage json.RawMessage func (j RawMessage) Value() (driver.Value, error) { @@ -76,25 +116,11 @@ func (j RawMessage) Value() (driver.Value, error) { return string(j), nil } -func (s *Store) initLastLog(ctx context.Context) { - s.once.Do(func() { - var err error - s.previousLog, err = s.GetLastLog(ctx) - if err != nil && !storageerrors.IsNotFoundError(err) { - panic(errors.Wrap(err, "reading last log")) - } - }) -} - -func (s *Store) InsertLogs(ctx context.Context, activeLogs []*core.ActiveLog) ([]*AppendedLog, error) { - recordMetrics := s.instrumentalized(ctx, "batch_logs") - defer recordMetrics() - - s.initLastLog(ctx) +func (s *Store) InsertLogs(ctx context.Context, activeLogs []*core.ActiveLog) error { txn, err := s.schema.BeginTx(ctx, &sql.TxOptions{}) if err != nil { - return nil, storageerrors.PostgresError(err) + return storageerrors.PostgresError(err) } // Beware: COPY query is not supported by bun if the pgx driver is used. @@ -104,58 +130,45 @@ func (s *Store) InsertLogs(ctx context.Context, activeLogs []*core.ActiveLog) ([ "id", "type", "hash", "date", "data", "idempotency_key", )) if err != nil { - return nil, storageerrors.PostgresError(err) + return storageerrors.PostgresError(err) } ls := make([]LogsV2, len(activeLogs)) - ret := make([]*AppendedLog, len(activeLogs)) for i, activeLog := range activeLogs { data, err := json.Marshal(activeLog.Data) if err != nil { - return nil, errors.Wrap(err, "marshaling log data") + return errors.Wrap(err, "marshaling log data") } - persistentLog := activeLog.ComputePersistentLog(s.previousLog) ls[i] = LogsV2{ - ID: persistentLog.ID, - Type: int16(persistentLog.Type), - Hash: persistentLog.Hash, - Date: persistentLog.Date, + ID: activeLog.ChainedLog.ID, + Type: int16(activeLog.ChainedLog.Type), + Hash: activeLog.ChainedLog.Hash, + Date: activeLog.ChainedLog.Date, Data: data, - IdempotencyKey: persistentLog.IdempotencyKey, - } - ret[i] = &AppendedLog{ - ActiveLog: activeLog, - PersistedLog: persistentLog, + IdempotencyKey: activeLog.ChainedLog.IdempotencyKey, } - s.previousLog = persistentLog - _, err = stmt.Exec(ls[i].ID, ls[i].Type, ls[i].Hash, ls[i].Date, RawMessage(ls[i].Data), persistentLog.IdempotencyKey) + _, err = stmt.Exec(ls[i].ID, ls[i].Type, ls[i].Hash, ls[i].Date, RawMessage(ls[i].Data), activeLog.ChainedLog.IdempotencyKey) if err != nil { - return nil, storageerrors.PostgresError(err) + return storageerrors.PostgresError(err) } } _, err = stmt.Exec() if err != nil { - return nil, storageerrors.PostgresError(err) + return storageerrors.PostgresError(err) } err = stmt.Close() if err != nil { - return nil, storageerrors.PostgresError(err) + return storageerrors.PostgresError(err) } - return ret, storageerrors.PostgresError(txn.Commit()) + return storageerrors.PostgresError(txn.Commit()) } -func (s *Store) GetLastLog(ctx context.Context) (*core.PersistedLog, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_last_log") - defer recordMetrics() - +func (s *Store) GetLastLog(ctx context.Context) (*core.ChainedLog, error) { raw := &LogsV2{} err := s.schema.NewSelect(LogTableName). Model(raw). @@ -170,13 +183,7 @@ func (s *Store) GetLastLog(ctx context.Context) (*core.PersistedLog, error) { return &l, nil } -func (s *Store) GetLogs(ctx context.Context, q LogsQuery) (*api.Cursor[core.PersistedLog], error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_logs") - defer recordMetrics() - +func (s *Store) GetLogs(ctx context.Context, q LogsQuery) (*api.Cursor[core.ChainedLog], error) { cursor, err := UsingColumn[LogsQueryFilters, LogsV2](ctx, s.buildLogsQuery, ColumnPaginatedQuery[LogsQueryFilters](q), @@ -203,49 +210,23 @@ func (s *Store) buildLogsQuery(q LogsQueryFilters, models *[]LogsV2) *bun.Select return sb } -func (s *Store) getNextLogID(ctx context.Context, sq interface { - NewSelect(string) *bun.SelectQuery -}) (uint64, error) { +func (s *Store) GetNextLogID(ctx context.Context) (uint64, error) { var logID uint64 - err := sq. - NewSelect(LogIngestionTableName). - Model((*LogsIngestion)(nil)). - Column("log_id"). + err := s.schema. + NewSelect(LogTableName). + ColumnExpr("min(id)"). + Where("projected = FALSE"). Limit(1). Scan(ctx, &logID) if err != nil { return 0, storageerrors.PostgresError(err) } - return logID, nil } -func (s *Store) GetNextLogID(ctx context.Context) (uint64, error) { - if !s.isInitialized { - return 0, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_next_log_id") - defer recordMetrics() - - return s.getNextLogID(ctx, &s.schema) -} - -func (s *Store) ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core.PersistedLog, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "read_logs_starting_from_id") - defer recordMetrics() - - return s.readLogsRange(ctx, &s.schema, idMin, idMax) -} - -func (s *Store) readLogsRange(ctx context.Context, exec interface { - NewSelect(tableName string) *bun.SelectQuery -}, idMin, idMax uint64) ([]core.PersistedLog, error) { - +func (s *Store) ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core.ChainedLog, error) { rawLogs := make([]LogsV2, 0) - err := exec. + err := s.schema. NewSelect(LogTableName). Where("id >= ?", idMin). Where("id < ?", idMax). @@ -258,32 +239,7 @@ func (s *Store) readLogsRange(ctx context.Context, exec interface { return collectionutils.Map(rawLogs, LogsV2.toCore), nil } -func (s *Store) UpdateNextLogID(ctx context.Context, id uint64) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "update_next_log_id") - defer recordMetrics() - - _, err := s.schema. - NewInsert(LogIngestionTableName). - Model(&LogsIngestion{ - LogId: id, - }). - On("CONFLICT (onerow_id) DO UPDATE"). - Set("log_id = EXCLUDED.log_id"). - Exec(ctx) - - return storageerrors.PostgresError(err) -} - -func (s *Store) ReadLastLogWithType(ctx context.Context, logTypes ...core.LogType) (*core.PersistedLog, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "read_last_log_with_type") - defer recordMetrics() - +func (s *Store) ReadLastLogWithType(ctx context.Context, logTypes ...core.LogType) (*core.ChainedLog, error) { raw := &LogsV2{} err := s.schema. NewSelect(LogTableName). @@ -300,13 +256,7 @@ func (s *Store) ReadLastLogWithType(ctx context.Context, logTypes ...core.LogTyp return &ret, nil } -func (s *Store) ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.PersistedLog, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "read_log_for_created_transaction_with_reference") - defer recordMetrics() - +func (s *Store) ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.ChainedLog, error) { raw := &LogsV2{} err := s.schema.NewSelect(LogTableName). Model(raw). @@ -322,13 +272,7 @@ func (s *Store) ReadLogForCreatedTransactionWithReference(ctx context.Context, r return &l, nil } -func (s *Store) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "read_log_for_created_transaction") - defer recordMetrics() - +func (s *Store) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.ChainedLog, error) { raw := &LogsV2{} err := s.schema.NewSelect(LogTableName). Model(raw). @@ -344,13 +288,7 @@ func (s *Store) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) ( return &l, nil } -func (s *Store) ReadLogForRevertedTransaction(ctx context.Context, txID uint64) (*core.PersistedLog, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "read_log_for_reverted_transaction") - defer recordMetrics() - +func (s *Store) ReadLogForRevertedTransaction(ctx context.Context, txID uint64) (*core.ChainedLog, error) { raw := &LogsV2{} err := s.schema.NewSelect(LogTableName). Model(raw). @@ -366,13 +304,7 @@ func (s *Store) ReadLogForRevertedTransaction(ctx context.Context, txID uint64) return &l, nil } -func (s *Store) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*core.PersistedLog, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "read_log_with_idempotency_key") - defer recordMetrics() - +func (s *Store) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*core.ChainedLog, error) { raw := &LogsV2{} err := s.schema.NewSelect(LogTableName). Model(raw). @@ -388,47 +320,69 @@ func (s *Store) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*cor return &l, nil } -type LogsQueryFilters struct { - EndTime core.Time `json:"endTime"` - StartTime core.Time `json:"startTime"` +func (s *Store) MarkedLogsAsProjected(ctx context.Context, id uint64) error { + _, err := s.schema.NewUpdate(LogTableName). + Where("id = ?", id). + Set("projected = TRUE"). + Exec(ctx) + return storageerrors.PostgresError(err) } -type LogsQuery ColumnPaginatedQuery[LogsQueryFilters] +func (s *Store) GetBalanceFromLogs(ctx context.Context, address, asset string) (*big.Int, error) { + selectLogsForExistingAccount := s.schema. + NewSelect(LogTableName). + Model(&LogsV2{}). + Where(fmt.Sprintf(`data->'transaction'->'postings' @> '[{"destination": "%s", "asset": "%s"}]' OR data->'transaction'->'postings' @> '[{"source": "%s", "asset": "%s"}]'`, address, asset, address, asset)) -func NewLogsQuery() LogsQuery { - return LogsQuery{ - PageSize: QueryDefaultPageSize, - Column: "id", - Order: OrderDesc, - Filters: LogsQueryFilters{}, - } -} + selectPostings := s.schema.IDB.NewSelect(). + TableExpr(`(` + selectLogsForExistingAccount.String() + `) as logs`). + ColumnExpr("jsonb_array_elements(logs.data->'transaction'->'postings') as postings") -func (a LogsQuery) WithPaginationID(id uint64) LogsQuery { - a.PaginationID = &id - return a -} + selectBalances := s.schema.IDB.NewSelect(). + TableExpr(`(` + selectPostings.String() + `) as postings`). + ColumnExpr(fmt.Sprintf("SUM(CASE WHEN (postings.postings::jsonb)->>'source' = '%s' THEN -((((postings.postings::jsonb)->'amount')::numeric)) ELSE ((postings.postings::jsonb)->'amount')::numeric END)", address)) -func (l LogsQuery) WithPageSize(pageSize uint64) LogsQuery { - if pageSize != 0 { - l.PageSize = pageSize + row := s.schema.IDB.QueryRowContext(ctx, selectBalances.String()) + if row.Err() != nil { + return nil, row.Err() } - return l + var balance *bunbig.Int + if err := row.Scan(&balance); err != nil { + return nil, err + } + return (*big.Int)(balance), nil } -func (l LogsQuery) WithStartTimeFilter(start core.Time) LogsQuery { - if !start.IsZero() { - l.Filters.StartTime = start +func (s *Store) GetMetadataFromLogs(ctx context.Context, address, key string) (string, error) { + l := LogsV2{} + if err := s.schema.NewSelect(LogTableName). + Model(&l). + Order("id DESC"). + WhereOr( + "type = ? AND data->>'targetId' = ? AND data->>'targetType' = ? AND "+fmt.Sprintf("data->'metadata' ? '%s'", key), + core.SetMetadataLogType, address, core.MetaTargetTypeAccount, + ). + WhereOr( + "type = ? AND "+fmt.Sprintf("data->'accountMetadata'->'%s' ? '%s'", address, key), + core.NewTransactionLogType, + ). + Limit(1). + Scan(ctx); err != nil { + return "", storageerrors.PostgresError(err) } - return l -} - -func (l LogsQuery) WithEndTimeFilter(end core.Time) LogsQuery { - if !end.IsZero() { - l.Filters.EndTime = end + payload, err := core.HydrateLog(core.LogType(l.Type), l.Data) + if err != nil { + panic(errors.Wrap(err, "hydrating log data")) } - return l + switch payload := payload.(type) { + case core.NewTransactionLogPayload: + return payload.AccountMetadata[address][key], nil + case core.SetMetadataLogPayload: + return payload.Metadata[key], nil + default: + panic("should not happen") + } } diff --git a/components/ledger/pkg/storage/ledgerstore/logs_test.go b/components/ledger/pkg/storage/ledgerstore/logs_test.go index 94eac03d43..9dc48605b4 100644 --- a/components/ledger/pkg/storage/ledgerstore/logs_test.go +++ b/components/ledger/pkg/storage/ledgerstore/logs_test.go @@ -2,12 +2,13 @@ package ledgerstore_test import ( "context" + "fmt" "math/big" "testing" "time" "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/storage/errors" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/stretchr/testify/require" @@ -16,12 +17,57 @@ import ( func TestGetLastLog(t *testing.T) { t.Parallel() store := newLedgerStore(t) + now := core.Now() lastLog, err := store.GetLastLog(context.Background()) - require.True(t, errors.IsNotFoundError(err)) + require.True(t, storage.IsNotFoundError(err)) require.Nil(t, lastLog) + tx1 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + TransactionData: core.TransactionData{ + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: big.NewInt(100), + Asset: "USD", + }, + }, + Reference: "tx1", + Timestamp: now.Add(-3 * time.Hour), + }, + }, + PostCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(100), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(100), + Output: big.NewInt(0), + }, + }, + }, + PreCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(0), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(0), + }, + }, + }, + } - logTx := core.NewTransactionLog(&tx1.Transaction, nil) + logTx := core.NewTransactionLog(&tx1.Transaction, nil).ChainLog(nil) appendLog(t, store, logTx) lastLog, err = store.GetLastLog(context.Background()) @@ -45,7 +91,7 @@ func TestReadLogForCreatedTransactionWithReference(t *testing.T) { WithReference("ref"), map[string]metadata.Metadata{}, ) - persistedLog := appendLog(t, store, logTx) + persistedLog := appendLog(t, store, logTx.ChainLog(nil)) lastLog, err := store.ReadLogForCreatedTransactionWithReference(context.Background(), "ref") require.NoError(t, err) @@ -61,7 +107,7 @@ func TestReadLogForRevertedTransaction(t *testing.T) { core.Now(), 0, core.NewTransaction(), - )) + ).ChainLog(nil)) lastLog, err := store.ReadLogForRevertedTransaction(context.Background(), 0) require.NoError(t, err) @@ -81,12 +127,12 @@ func TestReadLogForCreatedTransaction(t *testing.T) { WithReference("ref"), map[string]metadata.Metadata{}, ) - persistedLog := appendLog(t, store, logTx) + chainedLog := appendLog(t, store, logTx.ChainLog(nil)) - lastLog, err := store.ReadLogForCreatedTransaction(context.Background(), 0) + logFromDB, err := store.ReadLogForCreatedTransaction(context.Background(), 0) require.NoError(t, err) - require.NotNil(t, lastLog) - require.Equal(t, *persistedLog, *lastLog) + require.NotNil(t, logFromDB) + require.Equal(t, *chainedLog, *logFromDB) } func TestReadLogWithIdempotencyKey(t *testing.T) { @@ -102,7 +148,7 @@ func TestReadLogWithIdempotencyKey(t *testing.T) { ) log := logTx.WithIdempotencyKey("test") - ret := appendLog(t, store, log) + ret := appendLog(t, store, log.ChainLog(nil)) lastLog, err := store.ReadLogWithIdempotencyKey(context.Background(), "test") require.NoError(t, err) @@ -113,9 +159,151 @@ func TestReadLogWithIdempotencyKey(t *testing.T) { func TestGetLogs(t *testing.T) { t.Parallel() store := newLedgerStore(t) + now := core.Now() + + tx1 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + TransactionData: core.TransactionData{ + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: big.NewInt(100), + Asset: "USD", + }, + }, + Reference: "tx1", + Timestamp: now.Add(-3 * time.Hour), + }, + }, + PostCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(100), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(100), + Output: big.NewInt(0), + }, + }, + }, + PreCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(0), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(0), + }, + }, + }, + } + tx2 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 1, + TransactionData: core.TransactionData{ + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: big.NewInt(100), + Asset: "USD", + }, + }, + Reference: "tx2", + Timestamp: now.Add(-2 * time.Hour), + }, + }, + PostCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(200), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(200), + Output: big.NewInt(0), + }, + }, + }, + PreCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(100), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(100), + Output: big.NewInt(0), + }, + }, + }, + } + tx3 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 2, + TransactionData: core.TransactionData{ + Postings: []core.Posting{ + { + Source: "central_bank", + Destination: "users:1", + Amount: big.NewInt(1), + Asset: "USD", + }, + }, + Reference: "tx3", + Metadata: metadata.Metadata{ + "priority": "high", + }, + Timestamp: now.Add(-1 * time.Hour), + }, + }, + PreCommitVolumes: core.AccountsAssetsVolumes{ + "central_bank": { + "USD": { + Input: big.NewInt(200), + Output: big.NewInt(0), + }, + }, + "users:1": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(0), + }, + }, + }, + PostCommitVolumes: core.AccountsAssetsVolumes{ + "central_bank": { + "USD": { + Input: big.NewInt(200), + Output: big.NewInt(1), + }, + }, + "users:1": { + "USD": { + Input: big.NewInt(1), + Output: big.NewInt(0), + }, + }, + }, + } + var previousLog *core.ChainedLog for _, tx := range []core.ExpandedTransaction{tx1, tx2, tx3} { - appendLog(t, store, core.NewTransactionLog(&tx.Transaction, nil)) + newLog := core.NewTransactionLog(&tx.Transaction, nil).ChainLog(previousLog) + appendLog(t, store, newLog) + previousLog = newLog } cursor, err := store.GetLogs(context.Background(), ledgerstore.NewLogsQuery()) @@ -144,3 +332,86 @@ func TestGetLogs(t *testing.T) { require.Len(t, cursor.Data, 1) require.Equal(t, uint64(1), cursor.Data[0].ID) } + +func TestGetBalanceFromLogs(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + + const ( + batchNumber = 100 + batchSize = 10 + input = 100 + output = 10 + ) + + logs := make([]*core.ActiveLog, 0) + var previousLog *core.ChainedLog + for i := 0; i < batchNumber; i++ { + for j := 0; j < batchSize; j++ { + activeLog := core.NewActiveLog(core.NewTransactionLog( + core.NewTransaction().WithPostings( + core.NewPosting("world", fmt.Sprintf("account:%d", j), "EUR/2", big.NewInt(input)), + core.NewPosting(fmt.Sprintf("account:%d", j), "starbucks", "EUR/2", big.NewInt(output)), + ).WithID(uint64(i*batchSize+j)), + map[string]metadata.Metadata{}, + ).ChainLog(previousLog)) + logs = append(logs, activeLog) + previousLog = activeLog.ChainedLog + } + } + err := store.InsertLogs(context.Background(), logs) + require.NoError(t, err) + + balance, err := store.GetBalanceFromLogs(context.Background(), "account:1", "EUR/2") + require.NoError(t, err) + require.Equal(t, big.NewInt((input-output)*batchNumber), balance) +} + +func TestGetMetadataFromLogs(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + + logs := make([]*core.ActiveLog, 0) + logs = append(logs, core.NewActiveLog(core.NewTransactionLog( + core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "EUR/2", big.NewInt(100)), + core.NewPosting("bank", "starbucks", "EUR/2", big.NewInt(10)), + ), + map[string]metadata.Metadata{}, + ).ChainLog(nil))) + logs = append(logs, core.NewActiveLog(core.NewSetMetadataLog(core.Now(), core.SetMetadataLogPayload{ + TargetType: core.MetaTargetTypeAccount, + TargetID: "bank", + Metadata: metadata.Metadata{ + "foo": "bar", + }, + }).ChainLog(logs[0].ChainedLog))) + logs = append(logs, core.NewActiveLog(core.NewTransactionLog( + core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "EUR/2", big.NewInt(100)), + core.NewPosting("bank", "starbucks", "EUR/2", big.NewInt(10)), + ).WithID(1), + map[string]metadata.Metadata{}, + ).ChainLog(logs[1].ChainedLog))) + logs = append(logs, core.NewActiveLog(core.NewSetMetadataLog(core.Now(), core.SetMetadataLogPayload{ + TargetType: core.MetaTargetTypeAccount, + TargetID: "bank", + Metadata: metadata.Metadata{ + "role": "admin", + }, + }).ChainLog(logs[2].ChainedLog))) + logs = append(logs, core.NewActiveLog(core.NewTransactionLog( + core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "EUR/2", big.NewInt(100)), + core.NewPosting("bank", "starbucks", "EUR/2", big.NewInt(10)), + ).WithID(2), + map[string]metadata.Metadata{}, + ).ChainLog(logs[3].ChainedLog))) + + err := store.InsertLogs(context.Background(), logs) + require.NoError(t, err) + + metadata, err := store.GetMetadataFromLogs(context.Background(), "bank", "foo") + require.NoError(t, err) + require.Equal(t, "bar", metadata) +} diff --git a/components/ledger/pkg/storage/ledgerstore/logs_worker.go b/components/ledger/pkg/storage/ledgerstore/logs_worker.go index 559b49f7d6..2f51f3ccd6 100644 --- a/components/ledger/pkg/storage/ledgerstore/logs_worker.go +++ b/components/ledger/pkg/storage/ledgerstore/logs_worker.go @@ -4,26 +4,16 @@ import ( "context" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + "github.com/formancehq/stack/libs/go-libs/collectionutils" + "github.com/formancehq/stack/libs/go-libs/logging" ) -type AppendedLog struct { - ActiveLog *core.ActiveLog - PersistedLog *core.PersistedLog -} - type pendingLog struct { *core.LogPersistenceTracker log *core.ActiveLog } func (s *Store) AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "append_log") - defer recordMetrics() - ret := core.NewLogPersistenceTracker(log) select { @@ -42,26 +32,29 @@ func (s *Store) processPendingLogs(ctx context.Context, pendingLogs []pendingLog for _, holder := range pendingLogs { models = append(models, holder.log) } - appendedLogs, err := s.InsertLogs(ctx, models) + err := s.InsertLogs(ctx, models) if err != nil { panic(err) } - for i, holder := range pendingLogs { - holder.Resolve(appendedLogs[i].PersistedLog) + for _, holder := range pendingLogs { + holder.Resolve() } for _, f := range s.onLogsWrote { - f(appendedLogs) + f(collectionutils.Map(pendingLogs, func(from pendingLog) *core.ActiveLog { + return from.log + })) } } func (s *Store) Run(ctx context.Context) { writeLoopStopped := make(chan struct{}) effectiveSendChannel := make(chan []pendingLog) + stopped := make(chan struct{}) go func() { defer close(writeLoopStopped) for { select { - case <-s.stopped: + case <-stopped: return case pendingLogs := <-effectiveSendChannel: s.processPendingLogs(ctx, pendingLogs) @@ -72,15 +65,18 @@ func (s *Store) Run(ctx context.Context) { var ( sendChannel chan []pendingLog bufferedPendingLogs = make([]pendingLog, 0) + logger = logging.FromContext(ctx) ) for { select { case ch := <-s.stopChan: - close(s.stopped) + logger.Debugf("Terminating store worker, waiting end of write loop") + close(stopped) <-writeLoopStopped - if len(bufferedPendingLogs) > 0 { - s.processPendingLogs(ctx, bufferedPendingLogs) - } + logger.Debugf("Write loop terminated, store properly closed") + //if len(bufferedPendingLogs) > 0 { + // s.processPendingLogs(ctx, bufferedPendingLogs) + //} close(ch) return @@ -95,15 +91,20 @@ func (s *Store) Run(ctx context.Context) { } func (s *Store) Stop(ctx context.Context) error { + logging.FromContext(ctx).Info("Close store") ch := make(chan struct{}) select { case <-ctx.Done(): + logging.FromContext(ctx).Errorf("Unable to close store: %s", ctx.Err()) return ctx.Err() case s.stopChan <- ch: + logging.FromContext(ctx).Debugf("Signal sent, waiting response") select { case <-ch: + logging.FromContext(ctx).Info("Store closed") return nil case <-ctx.Done(): + logging.FromContext(ctx).Errorf("Unable to close store: %s", ctx.Err()) return ctx.Err() } } diff --git a/components/ledger/pkg/storage/ledgerstore/main_test.go b/components/ledger/pkg/storage/ledgerstore/main_test.go index 97fbdd9284..da7af7453a 100644 --- a/components/ledger/pkg/storage/ledgerstore/main_test.go +++ b/components/ledger/pkg/storage/ledgerstore/main_test.go @@ -2,19 +2,22 @@ package ledgerstore_test import ( "context" + "database/sql" + "fmt" "os" + "strings" "testing" "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/ledger/pkg/storage/ledgerstore" _ "github.com/formancehq/ledger/pkg/storage/ledgerstore/migrates/0-init-schema" - "github.com/formancehq/ledger/pkg/storage/schema" - "github.com/formancehq/ledger/pkg/storage/utils" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/uptrace/bun" ) func TestMain(m *testing.M) { @@ -34,16 +37,19 @@ func newLedgerStore(t *testing.T) *ledgerstore.Store { t.Helper() pgServer := pgtesting.NewPostgresDatabase(t) - db, err := utils.OpenSQLDB(utils.ConnectionOptions{ + db, err := storage.OpenSQLDB(storage.ConnectionOptions{ DatabaseSourceName: pgServer.ConnString(), Debug: testing.Verbose(), - }) + Trace: testing.Verbose(), + }, + //&explainHook{}, + ) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, db.Close()) }) - driver := storage.NewDriver("postgres", schema.NewPostgresDB(db), ledgerstore.DefaultStoreConfig) + driver := driver.New("postgres", storage.NewDatabase(db), ledgerstore.DefaultStoreConfig) require.NoError(t, driver.Initialize(context.Background())) ledgerStore, err := driver.CreateLedgerStore(context.Background(), uuid.NewString()) require.NoError(t, err) @@ -54,9 +60,46 @@ func newLedgerStore(t *testing.T) *ledgerstore.Store { return ledgerStore } -func appendLog(t *testing.T, store *ledgerstore.Store, log *core.Log) *core.PersistedLog { +func appendLog(t *testing.T, store *ledgerstore.Store, log *core.ChainedLog) *core.ChainedLog { ret, err := store.AppendLog(context.Background(), core.NewActiveLog(log)) <-ret.Done() require.NoError(t, err) - return ret.Result() + return ret.ActiveLog().ChainedLog +} + +type explainHook struct{} + +func (h *explainHook) AfterQuery(ctx context.Context, event *bun.QueryEvent) {} + +func (h *explainHook) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { + lowerQuery := strings.ToLower(event.Query) + if strings.HasPrefix(lowerQuery, "explain") || + strings.HasPrefix(lowerQuery, "create") || + strings.HasPrefix(lowerQuery, "begin") || + strings.HasPrefix(lowerQuery, "alter") || + strings.HasPrefix(lowerQuery, "rollback") || + strings.HasPrefix(lowerQuery, "commit") { + return ctx + } + + event.DB.RunInTx(context.Background(), &sql.TxOptions{}, func(ctx context.Context, tx bun.Tx) error { + rows, err := tx.Query("explain analyze " + event.Query) + if err != nil { + return err + } + defer rows.Next() + + for rows.Next() { + var line string + if err := rows.Scan(&line); err != nil { + return err + } + fmt.Println(line) + } + + return tx.Rollback() + + }) + + return ctx } diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go index e58a44f0a1..5421e76372 100644 --- a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go +++ b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go @@ -3,17 +3,15 @@ package initschema import ( "context" "database/sql" - "database/sql/driver" "encoding/json" "fmt" "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/ledger/aggregator" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + "github.com/formancehq/ledger/pkg/ledger/query" + "github.com/formancehq/ledger/pkg/opentelemetry/metrics" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/ledger/pkg/storage/migrations" - "github.com/formancehq/ledger/pkg/storage/schema" - "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/lib/pq" "github.com/pkg/errors" flag "github.com/spf13/pflag" @@ -54,8 +52,8 @@ type Log struct { func isLogTableExisting( ctx context.Context, - schema schema.Schema, - sqlTx *schema.Tx, + schema storage.Schema, + sqlTx *storage.Tx, ) (bool, error) { row := sqlTx.QueryRowContext(ctx, fmt.Sprintf(` SELECT EXISTS ( @@ -80,8 +78,8 @@ func isLogTableExisting( func readLogsRange( ctx context.Context, - schema schema.Schema, - sqlTx *schema.Tx, + schema storage.Schema, + sqlTx *storage.Tx, idMin, idMax uint64, ) ([]Log, error) { rawLogs := make([]Log, 0) @@ -120,13 +118,13 @@ func readLogsRange( return rawLogs, nil } -func (l *Log) ToLogsV2() (LogV2, error) { +func (l *Log) ToLogsV2() (ledgerstore.LogsV2, error) { logType, err := core.LogTypeFromString(l.Type) if err != nil { - return LogV2{}, errors.Wrap(err, "converting log type") + return ledgerstore.LogsV2{}, errors.Wrap(err, "converting log type") } - return LogV2{ + return ledgerstore.LogsV2{ ID: l.ID, Type: int16(logType), Hash: []byte(l.Hash), @@ -135,30 +133,11 @@ func (l *Log) ToLogsV2() (LogV2, error) { }, nil } -type LogV2 struct { - bun.BaseModel `bun:"logs_v2,alias:logs_v2"` - - ID uint64 `bun:"id,unique,type:bigint"` - Type int16 `bun:"type,type:smallint"` - Hash []byte `bun:"hash,type:varchar(256)"` - Date core.Time `bun:"date,type:timestamptz"` - Data []byte `bun:"data,type:bytea"` -} - -type RawMessage json.RawMessage - -func (j RawMessage) Value() (driver.Value, error) { - if j == nil { - return nil, nil - } - return string(j), nil -} - func batchLogs( ctx context.Context, - schema schema.Schema, - sqlTx *schema.Tx, - logs []LogV2, + schema storage.Schema, + sqlTx *storage.Tx, + logs []ledgerstore.LogsV2, ) error { txn, err := sqlTx.BeginTx(ctx, &sql.TxOptions{}) if err != nil { @@ -176,7 +155,7 @@ func batchLogs( } for _, l := range logs { - _, err = stmt.Exec(l.ID, l.Type, l.Hash, l.Date, RawMessage(l.Data)) + _, err = stmt.Exec(l.ID, l.Type, l.Hash, l.Date, ledgerstore.RawMessage(l.Data)) if err != nil { return err } @@ -197,9 +176,9 @@ func batchLogs( func cleanSchema( ctx context.Context, - schemaV1 schema.Schema, - schemaV2 schema.Schema, - sqlTx *schema.Tx, + schemaV1 storage.Schema, + schemaV2 storage.Schema, + sqlTx *storage.Tx, ) error { _, err := sqlTx.ExecContext(ctx, fmt.Sprintf(`ALTER SCHEMA "%s" RENAME TO "%s"`, schemaV1.Name(), schemaV1.Name()+oldSchemaRenameSuffix)) @@ -215,9 +194,9 @@ func cleanSchema( func migrateLogs( ctx context.Context, - schemaV1 schema.Schema, - schemaV2 schema.Schema, - sqlTx *schema.Tx, + schemaV1 storage.Schema, + schemaV2 storage.Schema, + sqlTx *storage.Tx, ) error { exists, err := isLogTableExisting(ctx, schemaV1, sqlTx) if err != nil { @@ -229,7 +208,7 @@ func migrateLogs( } var idMin uint64 - var idMax uint64 = idMin + batchSize + var idMax = idMin + batchSize for { logs, err := readLogsRange(ctx, schemaV1, sqlTx, idMin, idMax) if err != nil { @@ -240,7 +219,7 @@ func migrateLogs( break } - logsV2 := make([]LogV2, 0, len(logs)) + logsV2 := make([]ledgerstore.LogsV2, 0, len(logs)) for _, l := range logs { logV2, err := l.ToLogsV2() if err != nil { @@ -262,217 +241,10 @@ func migrateLogs( return nil } -func processLogs( - ctx context.Context, - store *ledgerstore.Store, - logs ...core.PersistedLog, -) error { - logsData, err := buildData(ctx, store, logs...) - if err != nil { - return errors.Wrap(err, "building data") - } - - if err := store.RunInTransaction(ctx, func(ctx context.Context, tx *ledgerstore.Store) error { - if len(logsData.ensureAccountsExist) > 0 { - if err := tx.EnsureAccountsExist(ctx, logsData.ensureAccountsExist); err != nil { - return errors.Wrap(err, "ensuring accounts exist") - } - } - if len(logsData.accountsToUpdate) > 0 { - if err := tx.UpdateAccountsMetadata(ctx, logsData.accountsToUpdate); err != nil { - return errors.Wrap(err, "updating accounts metadata") - } - } - - if len(logsData.transactionsToInsert) > 0 { - if err := tx.InsertTransactions(ctx, logsData.transactionsToInsert...); err != nil { - return errors.Wrap(err, "inserting transactions") - } - } - - if len(logsData.transactionsToUpdate) > 0 { - if err := tx.UpdateTransactionsMetadata(ctx, logsData.transactionsToUpdate...); err != nil { - return errors.Wrap(err, "updating transactions") - } - } - - if len(logsData.volumesToUpdate) > 0 { - return tx.UpdateVolumes(ctx, logsData.volumesToUpdate...) - } - - return nil - }); err != nil { - return err - } - - return nil -} - -type logsData struct { - accountsToUpdate []core.Account - ensureAccountsExist []string - transactionsToInsert []core.ExpandedTransaction - transactionsToUpdate []core.TransactionWithMetadata - volumesToUpdate []core.AccountsAssetsVolumes -} - -func buildData( - ctx context.Context, - store *ledgerstore.Store, - logs ...core.PersistedLog, -) (*logsData, error) { - logsData := &logsData{} - - volumeAggregator := aggregator.Volumes(store) - accountsToUpdate := make(map[string]metadata.Metadata) - transactionsToUpdate := make(map[uint64]metadata.Metadata) - - for _, log := range logs { - switch log.Type { - case core.NewTransactionLogType: - payload := log.Data.(core.NewTransactionLogPayload) - txVolumeAggregator, err := volumeAggregator.NextTxWithPostings(ctx, payload.Transaction.Postings...) - if err != nil { - return nil, err - } - - if payload.AccountMetadata != nil { - for account, metadata := range payload.AccountMetadata { - if m, ok := accountsToUpdate[account]; !ok { - accountsToUpdate[account] = metadata - } else { - for k, v := range metadata { - m[k] = v - } - } - } - } - - expandedTx := core.ExpandedTransaction{ - Transaction: *payload.Transaction, - PreCommitVolumes: txVolumeAggregator.PreCommitVolumes, - PostCommitVolumes: txVolumeAggregator.PostCommitVolumes, - } - - logsData.transactionsToInsert = append(logsData.transactionsToInsert, expandedTx) - - for account := range txVolumeAggregator.PostCommitVolumes { - logsData.ensureAccountsExist = append(logsData.ensureAccountsExist, account) - } - - logsData.volumesToUpdate = append(logsData.volumesToUpdate, txVolumeAggregator.PostCommitVolumes) - - case core.SetMetadataLogType: - setMetadata := log.Data.(core.SetMetadataLogPayload) - switch setMetadata.TargetType { - case core.MetaTargetTypeAccount: - addr := setMetadata.TargetID.(string) - if m, ok := accountsToUpdate[addr]; !ok { - accountsToUpdate[addr] = setMetadata.Metadata - } else { - for k, v := range setMetadata.Metadata { - m[k] = v - } - } - - case core.MetaTargetTypeTransaction: - id := setMetadata.TargetID.(uint64) - if m, ok := transactionsToUpdate[id]; !ok { - transactionsToUpdate[id] = setMetadata.Metadata - } else { - for k, v := range setMetadata.Metadata { - m[k] = v - } - } - } - - case core.RevertedTransactionLogType: - payload := log.Data.(core.RevertedTransactionLogPayload) - id := payload.RevertedTransactionID - metadata := core.RevertedMetadata(payload.RevertTransaction.ID) - if m, ok := transactionsToUpdate[id]; !ok { - transactionsToUpdate[id] = metadata - } else { - for k, v := range metadata { - m[k] = v - } - } - - txVolumeAggregator, err := volumeAggregator.NextTxWithPostings(ctx, payload.RevertTransaction.Postings...) - if err != nil { - return nil, errors.Wrap(err, "aggregating volumes") - } - - expandedTx := core.ExpandedTransaction{ - Transaction: *payload.RevertTransaction, - PreCommitVolumes: txVolumeAggregator.PreCommitVolumes, - PostCommitVolumes: txVolumeAggregator.PostCommitVolumes, - } - logsData.transactionsToInsert = append(logsData.transactionsToInsert, expandedTx) - } - } - - for account, metadata := range accountsToUpdate { - logsData.accountsToUpdate = append(logsData.accountsToUpdate, core.Account{ - Address: account, - Metadata: metadata, - }) - } - - for transaction, metadata := range transactionsToUpdate { - logsData.transactionsToUpdate = append(logsData.transactionsToUpdate, core.TransactionWithMetadata{ - ID: transaction, - Metadata: metadata, - }) - } - - return logsData, nil -} - -func initLedger( - ctx context.Context, - store *ledgerstore.Store, -) error { - if !store.IsInitialized() { - return nil - } - - lastReadLogID, err := store.GetNextLogID(ctx) - if err != nil && !storageerrors.IsNotFoundError(err) { - return errors.Wrap(err, "reading last log") - } - - for { - logs, err := store.ReadLogsRange(ctx, lastReadLogID, lastReadLogID+uint64(batchSize)) - if err != nil { - return errors.Wrap(err, "reading logs since last ID") - } - - if len(logs) == 0 { - // No logs, nothing to do - return nil - } - - if err := processLogs(ctx, store, logs...); err != nil { - return errors.Wrap(err, "processing logs") - } - - if err := store.UpdateNextLogID(ctx, logs[len(logs)-1].ID+1); err != nil { - return errors.Wrap(err, "updating last read log") - } - lastReadLogID = logs[len(logs)-1].ID + 1 - - if uint64(len(logs)) < batchSize { - // Nothing to do anymore, no need to read more logs - return nil - } - } -} - func UpgradeLogs( ctx context.Context, - schemaV1 schema.Schema, - sqlTx *schema.Tx, + schemaV1 storage.Schema, + sqlTx *storage.Tx, ) error { b := viper.GetUint64(LogsMigrationBatchSizeFlag) if b != 0 { @@ -485,8 +257,8 @@ func UpgradeLogs( } // Create schema v2 - schemaV2 := schema.NewSchema(sqlTx.Tx, schemaV1.Name()+"_v2_0_0") - store, err := ledgerstore.NewStore( + schemaV2 := storage.NewSchema(sqlTx.Tx, schemaV1.Name()+"_v2_0_0") + store, err := ledgerstore.New( schemaV2, func(ctx context.Context) error { return nil @@ -501,9 +273,9 @@ func UpgradeLogs( return errors.Wrap(err, "migrating logs") } - if err := initLedger(ctx, store); err != nil { - return errors.Wrap(err, "initializing ledger") - } + projector := query.NewProjector(store, query.NewNoOpMonitor(), metrics.NewNoOpRegistry()) + projector.Start(ctx) // Start block until logs are synced + projector.Stop(ctx) return cleanSchema(ctx, schemaV1, schemaV2, sqlTx) } diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql index 4118482cb1..42183747f1 100644 --- a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql +++ b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql @@ -1,66 +1,45 @@ --statement -CREATE SCHEMA IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0"; +create schema "VAR_LEDGER_NAME_v2_0_0"; --statement -CREATE FUNCTION "VAR_LEDGER_NAME_v2_0_0".meta_compare(metadata jsonb, value boolean, VARIADIC path text[]) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ BEGIN return jsonb_extract_path(metadata, variadic path)::bool = value::bool; EXCEPTION WHEN others THEN RAISE INFO 'Error Name: %', SQLERRM; RAISE INFO 'Error State: %', SQLSTATE; RETURN false; END $$; +create function "VAR_LEDGER_NAME_v2_0_0".meta_compare(metadata jsonb, value boolean, variadic path text[]) returns boolean + language plpgsql immutable + as $$ begin return jsonb_extract_path(metadata, variadic path)::bool = value::bool; exception when others then raise info 'Error Name: %', SQLERRM; raise info 'Error State: %', SQLSTATE; return false; END $$; --statement -CREATE FUNCTION "VAR_LEDGER_NAME_v2_0_0".meta_compare(metadata jsonb, value numeric, VARIADIC path text[]) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ BEGIN return jsonb_extract_path(metadata, variadic path)::numeric = value::numeric; EXCEPTION WHEN others THEN RAISE INFO 'Error Name: %', SQLERRM; RAISE INFO 'Error State: %', SQLSTATE; RETURN false; END $$; +create function "VAR_LEDGER_NAME_v2_0_0".meta_compare(metadata jsonb, value numeric, variadic path text[]) returns boolean + language plpgsql immutable + as $$ begin return jsonb_extract_path(metadata, variadic path)::numeric = value::numeric; exception when others then raise info 'Error Name: %', SQLERRM; raise info 'Error State: %', SQLSTATE; return false; END $$; --statement -CREATE FUNCTION "VAR_LEDGER_NAME_v2_0_0".meta_compare(metadata jsonb, value character varying, VARIADIC path text[]) RETURNS boolean - LANGUAGE plpgsql IMMUTABLE - AS $$ BEGIN return jsonb_extract_path_text(metadata, variadic path)::varchar = value::varchar; EXCEPTION WHEN others THEN RAISE INFO 'Error Name: %', SQLERRM; RAISE INFO 'Error State: %', SQLSTATE; RETURN false; END $$; +create function "VAR_LEDGER_NAME_v2_0_0".meta_compare(metadata jsonb, value character varying, variadic path text[]) returns boolean + language plpgsql immutable + as $$ begin return jsonb_extract_path_text(metadata, variadic path)::varchar = value::varchar; exception when others then raise info 'Error Name: %', SQLERRM; raise info 'Error State: %', SQLSTATE; return false; END $$; --statement -CREATE FUNCTION "VAR_LEDGER_NAME_v2_0_0".use_account_as_destination(postings jsonb, account character varying) RETURNS boolean - LANGUAGE sql - AS $_$ select bool_or(v.value::bool) from ( select jsonb_extract_path_text(jsonb_array_elements(postings), 'destination') ~ ('^' || account || '$') as value) as v; $_$; - ---statement -CREATE FUNCTION "VAR_LEDGER_NAME_v2_0_0".use_account_as_source(postings jsonb, account character varying) RETURNS boolean - LANGUAGE sql - AS $_$ select bool_or(v.value::bool) from ( select jsonb_extract_path_text(jsonb_array_elements(postings), 'source') ~ ('^' || account || '$') as value) as v; $_$; - ---statement -CREATE FUNCTION "VAR_LEDGER_NAME_v2_0_0".use_account(postings jsonb, account character varying) RETURNS boolean - LANGUAGE sql - AS $$ SELECT bool_or(v.value) from ( SELECT "VAR_LEDGER_NAME_v2_0_0".use_account_as_source(postings, account) AS value UNION SELECT "VAR_LEDGER_NAME_v2_0_0".use_account_as_destination(postings, account) AS value ) v $$; - ---statement -CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".accounts ( - address character varying NOT NULL, - metadata jsonb DEFAULT '{}'::jsonb, +create table "VAR_LEDGER_NAME_v2_0_0".accounts ( + address character varying not null, + address_json jsonb not null, + metadata jsonb default '{}'::jsonb, unique(address) ); --statement -CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".logs_ingestion ( - onerow_id boolean DEFAULT true NOT NULL, - log_id bigint, - - primary key (onerow_id) -); - ---statement -CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".logs_v2 ( +create table "VAR_LEDGER_NAME_v2_0_0".logs_v2 ( id bigint, type smallint, hash bytea, date timestamp with time zone, data jsonb, idempotency_key varchar(255), + projected boolean default false, unique(id) ); --statement -CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".migrations_v2 ( +create table "VAR_LEDGER_NAME_v2_0_0".migrations_v2 ( version character varying, date character varying, @@ -68,54 +47,87 @@ CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".migrations_v2 ( ); --statement -CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".transactions ( - id bigint unique, +create table "VAR_LEDGER_NAME_v2_0_0".transactions ( + id bigint unique primary key , "timestamp" timestamp with time zone not null, reference character varying unique, - metadata jsonb DEFAULT '{}'::jsonb, - pre_commit_volumes bytea, - post_commit_volumes bytea + metadata jsonb default '{}'::jsonb ); --statement -CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".postings ( - txid bigint references "VAR_LEDGER_NAME_v2_0_0".transactions(id), - amount bigint not null, - asset varchar not null, - source jsonb not null, - destination jsonb not null, - index int8, +create table "VAR_LEDGER_NAME_v2_0_0".moves ( + posting_index int8, + transaction_id bigint, + account varchar, + account_array jsonb not null, + asset varchar, + post_commit_input_value numeric, + post_commit_output_value numeric, + timestamp timestamp with time zone, + amount numeric not null, + is_source boolean, - primary key (txid, index) + primary key (transaction_id, posting_index, is_source) ); --statement -CREATE TABLE IF NOT EXISTS "VAR_LEDGER_NAME_v2_0_0".volumes ( - account character varying not null, - asset character varying not null, - input numeric not null, - output numeric not null, +create index logsv2_type on "VAR_LEDGER_NAME_v2_0_0".logs_v2 (type); - unique(account, asset) -); +--statement +create index logsv2_projected on "VAR_LEDGER_NAME_v2_0_0".logs_v2 (projected); + +--statement +create index logsv2_data on "VAR_LEDGER_NAME_v2_0_0".logs_v2 using gin (data); + +--statement +create index logsv2_new_transaction_postings on "VAR_LEDGER_NAME_v2_0_0".logs_v2 using gin ((data->'transaction'->'postings') jsonb_path_ops); + +--statement +create index logsv2_set_metadata on "VAR_LEDGER_NAME_v2_0_0".logs_v2 using btree (type, (data->>'targetId'), (data->>'targetType')); + +--statement +create index transactions_id_timestamp on "VAR_LEDGER_NAME_v2_0_0".transactions(id, timestamp); + +--statement +create index transactions_timestamp on "VAR_LEDGER_NAME_v2_0_0".transactions(timestamp); --statement -CREATE INDEX IF NOT EXISTS postings_dest ON "VAR_LEDGER_NAME_v2_0_0".postings USING gin (destination); +create index transactions_reference on "VAR_LEDGER_NAME_v2_0_0".transactions(reference); --statement -CREATE INDEX IF NOT EXISTS postings_src ON "VAR_LEDGER_NAME_v2_0_0".postings USING gin (source); +create index transactions_metadata on "VAR_LEDGER_NAME_v2_0_0".transactions using gin(metadata); --statement -CREATE INDEX IF NOT EXISTS logsv2_type ON "VAR_LEDGER_NAME_v2_0_0".logs_v2 (type); +create index moves_transaction_id on "VAR_LEDGER_NAME_v2_0_0".moves(transaction_id, posting_index); --statement -CREATE INDEX IF NOT EXISTS logsv2_data ON "VAR_LEDGER_NAME_v2_0_0".logs_v2 USING gin (data); +create index moves_account_array on "VAR_LEDGER_NAME_v2_0_0".moves using gin(account_array); --statement -CREATE INDEX IF NOT EXISTS postings_txid ON "VAR_LEDGER_NAME_v2_0_0".postings USING btree (txid); +create index moves_account on "VAR_LEDGER_NAME_v2_0_0".moves(account, asset, timestamp); --statement -CREATE INDEX IF NOT EXISTS logsv2_new_transaction_postings ON "VAR_LEDGER_NAME_v2_0_0".logs_v2 USING gin ((data->'transaction'->'postings') jsonb_path_ops); +create index moves_is_source on "VAR_LEDGER_NAME_v2_0_0".moves(account, is_source); --statement -CREATE INDEX IF NOT EXISTS logsv2_set_metadata ON "VAR_LEDGER_NAME_v2_0_0".logs_v2 USING btree (type, (data->>'targetId'), (data->>'targetType')); +create index accounts_address_json on "VAR_LEDGER_NAME_v2_0_0".accounts using GIN(address_json); + +--statement +create function "VAR_LEDGER_NAME_v2_0_0".first_agg (anyelement, anyelement) + returns anyelement + language sql immutable strict parallel safe as +'select $1'; + +--statement +create aggregate "VAR_LEDGER_NAME_v2_0_0".first (anyelement) ( + sfunc = "VAR_LEDGER_NAME_v2_0_0".first_agg +, stype = anyelement +, parallel = safe +); + +--statement +create aggregate "VAR_LEDGER_NAME_v2_0_0".aggregate_objects(jsonb) ( + sfunc = jsonb_concat, + stype = jsonb, + initcond = '{}' +); diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/1-optimized-accounts-volumes/postgres.sql b/components/ledger/pkg/storage/ledgerstore/migrates/1-optimized-accounts-volumes/postgres.sql deleted file mode 100644 index 812dcbdd33..0000000000 --- a/components/ledger/pkg/storage/ledgerstore/migrates/1-optimized-accounts-volumes/postgres.sql +++ /dev/null @@ -1,12 +0,0 @@ ---statement -alter table "VAR_LEDGER_NAME".accounts add column if not exists address_json jsonb; ---statement -UPDATE "VAR_LEDGER_NAME".accounts SET address_json = to_jsonb(string_to_array(address, ':')); ---statement -create index if not exists accounts_address_json on "VAR_LEDGER_NAME".accounts using GIN(address_json); ---statement -alter table "VAR_LEDGER_NAME".volumes add column if not exists account_json jsonb; ---statement -UPDATE "VAR_LEDGER_NAME".volumes SET account_json = to_jsonb(string_to_array(account, ':')); ---statement -create index if not exists volumes_account_json on "VAR_LEDGER_NAME".volumes using GIN(account_json); diff --git a/components/ledger/pkg/storage/ledgerstore/migration.go b/components/ledger/pkg/storage/ledgerstore/migration.go index d1fb48bd81..7b3bb7d541 100644 --- a/components/ledger/pkg/storage/ledgerstore/migration.go +++ b/components/ledger/pkg/storage/ledgerstore/migration.go @@ -5,7 +5,7 @@ import ( "embed" "github.com/formancehq/ledger/pkg/core" - sqlerrors "github.com/formancehq/ledger/pkg/storage/errors" + sqlerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/migrations" "github.com/pkg/errors" ) diff --git a/components/ledger/pkg/storage/ledgerstore/pagination_column.go b/components/ledger/pkg/storage/ledgerstore/pagination_column.go index b876f626b7..5d004c350a 100644 --- a/components/ledger/pkg/storage/ledgerstore/pagination_column.go +++ b/components/ledger/pkg/storage/ledgerstore/pagination_column.go @@ -6,8 +6,9 @@ import ( "reflect" "strings" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/api" + "github.com/formancehq/stack/libs/go-libs/pointer" "github.com/uptrace/bun" ) @@ -90,13 +91,13 @@ func UsingColumn[FILTERS any, ENTITY any](ctx context.Context, if hasMore { cp := query - cp.PaginationID = ptr(paginationIDs[len(paginationIDs)-2]) + cp.PaginationID = pointer.For(paginationIDs[len(paginationIDs)-2]) previous = &cp } } else { if hasMore { cp := query - cp.PaginationID = ptr(paginationIDs[len(paginationIDs)-1]) + cp.PaginationID = pointer.For(paginationIDs[len(paginationIDs)-1]) next = &cp } if query.PaginationID != nil { diff --git a/components/ledger/pkg/storage/ledgerstore/pagination_column_test.go b/components/ledger/pkg/storage/ledgerstore/pagination_column_test.go index b30264b871..7d2a805854 100644 --- a/components/ledger/pkg/storage/ledgerstore/pagination_column_test.go +++ b/components/ledger/pkg/storage/ledgerstore/pagination_column_test.go @@ -4,23 +4,22 @@ import ( "context" "testing" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/ledger/pkg/storage/utils" "github.com/formancehq/stack/libs/go-libs/pgtesting" + "github.com/formancehq/stack/libs/go-libs/pointer" "github.com/stretchr/testify/require" "github.com/uptrace/bun" ) -func ptr[T any](t T) *T { - return &t -} - func TestColumnPagination(t *testing.T) { + t.Parallel() pgServer := pgtesting.NewPostgresDatabase(t) - db, err := utils.OpenSQLDB(utils.ConnectionOptions{ + db, err := storage.OpenSQLDB(storage.ConnectionOptions{ DatabaseSourceName: pgServer.ConnString(), Debug: testing.Verbose(), + Trace: testing.Verbose(), }) require.NoError(t, err) @@ -65,9 +64,9 @@ func TestColumnPagination(t *testing.T) { expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(10)), + PaginationID: pointer.For(uint64(10)), Order: ledgerstore.OrderAsc, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), }, expectedNumberOfItems: 10, }, @@ -76,24 +75,24 @@ func TestColumnPagination(t *testing.T) { query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(10)), + PaginationID: pointer.For(uint64(10)), Order: ledgerstore.OrderAsc, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), }, expectedPrevious: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", Order: ledgerstore.OrderAsc, - Bottom: ptr(uint64(0)), - PaginationID: ptr(uint64(10)), + Bottom: pointer.For(uint64(0)), + PaginationID: pointer.For(uint64(10)), Reverse: true, }, expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(20)), + PaginationID: pointer.For(uint64(20)), Order: ledgerstore.OrderAsc, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), }, expectedNumberOfItems: 10, }, @@ -102,16 +101,16 @@ func TestColumnPagination(t *testing.T) { query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(90)), + PaginationID: pointer.For(uint64(90)), Order: ledgerstore.OrderAsc, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), }, expectedPrevious: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", Order: ledgerstore.OrderAsc, - PaginationID: ptr(uint64(90)), - Bottom: ptr(uint64(0)), + PaginationID: pointer.For(uint64(90)), + Bottom: pointer.For(uint64(0)), Reverse: true, }, expectedNumberOfItems: 10, @@ -125,9 +124,9 @@ func TestColumnPagination(t *testing.T) { }, expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(89)), + PaginationID: pointer.For(uint64(89)), Order: ledgerstore.OrderDesc, }, expectedNumberOfItems: 10, @@ -136,24 +135,24 @@ func TestColumnPagination(t *testing.T) { name: "desc second page using next cursor", query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(89)), + PaginationID: pointer.For(uint64(89)), Order: ledgerstore.OrderDesc, }, expectedPrevious: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(89)), + PaginationID: pointer.For(uint64(89)), Order: ledgerstore.OrderDesc, Reverse: true, }, expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(79)), + PaginationID: pointer.For(uint64(79)), Order: ledgerstore.OrderDesc, }, expectedNumberOfItems: 10, @@ -162,16 +161,16 @@ func TestColumnPagination(t *testing.T) { name: "desc last page using next cursor", query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(9)), + PaginationID: pointer.For(uint64(9)), Order: ledgerstore.OrderDesc, }, expectedPrevious: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(9)), + PaginationID: pointer.For(uint64(9)), Order: ledgerstore.OrderDesc, Reverse: true, }, @@ -181,17 +180,17 @@ func TestColumnPagination(t *testing.T) { name: "asc first page using previous cursor", query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), Column: "id", - PaginationID: ptr(uint64(10)), + PaginationID: pointer.For(uint64(10)), Order: ledgerstore.OrderAsc, Reverse: true, }, expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), Column: "id", - PaginationID: ptr(uint64(10)), + PaginationID: pointer.For(uint64(10)), Order: ledgerstore.OrderAsc, }, expectedNumberOfItems: 10, @@ -200,17 +199,17 @@ func TestColumnPagination(t *testing.T) { name: "desc first page using previous cursor", query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(89)), + PaginationID: pointer.For(uint64(89)), Order: ledgerstore.OrderDesc, Reverse: true, }, expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, - Bottom: ptr(uint64(99)), + Bottom: pointer.For(uint64(99)), Column: "id", - PaginationID: ptr(uint64(89)), + PaginationID: pointer.For(uint64(89)), Order: ledgerstore.OrderDesc, }, expectedNumberOfItems: 10, @@ -226,10 +225,10 @@ func TestColumnPagination(t *testing.T) { expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(20)), + PaginationID: pointer.For(uint64(20)), Order: ledgerstore.OrderAsc, Filters: true, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), }, expectedNumberOfItems: 10, }, @@ -238,26 +237,26 @@ func TestColumnPagination(t *testing.T) { query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(20)), + PaginationID: pointer.For(uint64(20)), Order: ledgerstore.OrderAsc, Filters: true, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), }, expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(40)), + PaginationID: pointer.For(uint64(40)), Order: ledgerstore.OrderAsc, Filters: true, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), }, expectedPrevious: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(20)), + PaginationID: pointer.For(uint64(20)), Order: ledgerstore.OrderAsc, Filters: true, - Bottom: ptr(uint64(0)), + Bottom: pointer.For(uint64(0)), Reverse: true, }, expectedNumberOfItems: 10, @@ -273,10 +272,10 @@ func TestColumnPagination(t *testing.T) { expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(78)), + PaginationID: pointer.For(uint64(78)), Order: ledgerstore.OrderDesc, Filters: true, - Bottom: ptr(uint64(98)), + Bottom: pointer.For(uint64(98)), }, expectedNumberOfItems: 10, }, @@ -285,26 +284,26 @@ func TestColumnPagination(t *testing.T) { query: ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(78)), + PaginationID: pointer.For(uint64(78)), Order: ledgerstore.OrderDesc, Filters: true, - Bottom: ptr(uint64(98)), + Bottom: pointer.For(uint64(98)), }, expectedNext: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(58)), + PaginationID: pointer.For(uint64(58)), Order: ledgerstore.OrderDesc, Filters: true, - Bottom: ptr(uint64(98)), + Bottom: pointer.For(uint64(98)), }, expectedPrevious: &ledgerstore.ColumnPaginatedQuery[bool]{ PageSize: 10, Column: "id", - PaginationID: ptr(uint64(78)), + PaginationID: pointer.For(uint64(78)), Order: ledgerstore.OrderDesc, Filters: true, - Bottom: ptr(uint64(98)), + Bottom: pointer.For(uint64(98)), Reverse: true, }, expectedNumberOfItems: 10, diff --git a/components/ledger/pkg/storage/ledgerstore/pagination_offset.go b/components/ledger/pkg/storage/ledgerstore/pagination_offset.go index 99705c5a14..b20ab7be9e 100644 --- a/components/ledger/pkg/storage/ledgerstore/pagination_offset.go +++ b/components/ledger/pkg/storage/ledgerstore/pagination_offset.go @@ -3,37 +3,18 @@ package ledgerstore import ( "context" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" "github.com/formancehq/stack/libs/go-libs/api" "github.com/uptrace/bun" ) -type scanner[T any] func(t *T, scanner interface { - Scan(args ...any) error -}) error - -func UsingOffset[Q any, T any](ctx context.Context, sb *bun.SelectQuery, query OffsetPaginatedQuery[Q], fn scanner[T]) (*api.Cursor[T], error) { +func UsingOffset[Q any, T any](ctx context.Context, sb *bun.SelectQuery, query OffsetPaginatedQuery[Q]) (*api.Cursor[T], error) { ret := make([]T, 0) sb = sb.Offset(int(query.Offset)) sb = sb.Limit(int(query.PageSize) + 1) - rows, err := sb.Rows(ctx) - if err != nil { - return nil, storageerrors.PostgresError(err) - } - defer rows.Close() - - for rows.Next() { - var t T - if err := fn(&t, rows); err != nil { - return nil, err - } - ret = append(ret, t) - } - - if rows.Err() != nil { - return nil, storageerrors.PostgresError(err) + if err := sb.Scan(ctx, &ret); err != nil { + return nil, err } var previous, next *OffsetPaginatedQuery[Q] diff --git a/components/ledger/pkg/storage/ledgerstore/pagination_offset_test.go b/components/ledger/pkg/storage/ledgerstore/pagination_offset_test.go index 32b8bd4e87..ddd53961cd 100644 --- a/components/ledger/pkg/storage/ledgerstore/pagination_offset_test.go +++ b/components/ledger/pkg/storage/ledgerstore/pagination_offset_test.go @@ -4,18 +4,20 @@ import ( "context" "testing" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/ledger/pkg/storage/utils" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" ) func TestOffsetPagination(t *testing.T) { + t.Parallel() pgServer := pgtesting.NewPostgresDatabase(t) - db, err := utils.OpenSQLDB(utils.ConnectionOptions{ + db, err := storage.OpenSQLDB(storage.ConnectionOptions{ DatabaseSourceName: pgServer.ConnString(), Debug: testing.Verbose(), + Trace: testing.Verbose(), }) require.NoError(t, err) @@ -139,13 +141,10 @@ func TestOffsetPagination(t *testing.T) { if tc.query.Filters { query = query.Where("pair = ?", true) } - cursor, err := ledgerstore.UsingOffset( + cursor, err := ledgerstore.UsingOffset[bool, model]( context.Background(), query, - tc.query, - func(t *model, scanner interface{ Scan(args ...any) error }) error { - return scanner.Scan(&t.ID) - }) + tc.query) require.NoError(t, err) if tc.expectedNext == nil { diff --git a/components/ledger/pkg/storage/ledgerstore/store.go b/components/ledger/pkg/storage/ledgerstore/store.go index d86f437e50..46d26f6ee3 100644 --- a/components/ledger/pkg/storage/ledgerstore/store.go +++ b/components/ledger/pkg/storage/ledgerstore/store.go @@ -2,18 +2,13 @@ package ledgerstore import ( "context" - "database/sql" "sync" - "time" "github.com/formancehq/ledger/pkg/core" - sqlerrors "github.com/formancehq/ledger/pkg/storage/errors" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/migrations" - "github.com/formancehq/ledger/pkg/storage/opentelemetry/metrics" - "github.com/formancehq/ledger/pkg/storage/schema" _ "github.com/jackc/pgx/v5/stdlib" "github.com/pkg/errors" - "go.opentelemetry.io/otel/attribute" ) const ( @@ -33,22 +28,19 @@ var ( } ) -type OnLogWrote func([]*AppendedLog) +type OnLogWrote func([]*core.ActiveLog) type Store struct { - schema schema.Schema - metricsRegistry *metrics.SQLStorageMetricsRegistry - storeConfig StoreConfig - onDelete func(ctx context.Context) error + schema storage.Schema + storeConfig StoreConfig + onDelete func(ctx context.Context) error - previousLog *core.PersistedLog - once sync.Once + once sync.Once isInitialized bool writeChannel chan pendingLog stopChan chan chan struct{} - stopped chan struct{} onLogsWrote []OnLogWrote } @@ -62,7 +54,7 @@ var ( } ) -func (s *Store) Schema() schema.Schema { +func (s *Store) Schema() storage.Schema { return s.schema } @@ -95,76 +87,20 @@ func (s *Store) IsInitialized() bool { return s.isInitialized } -func (s *Store) RunInTransaction(ctx context.Context, f func(ctx context.Context, store *Store) error) error { - tx, err := s.schema.BeginTx(ctx, &sql.TxOptions{}) - if err != nil { - return sqlerrors.PostgresError(err) - } - - // Create a fake store to use the tx instead of the bun.DB struct - // TODO(polo): it can be heavy to create and drop store for each transaction - // since we're creating workers etc... - newStore, err := NewStore( - schema.NewSchema(tx.Tx, s.schema.Name()), - func(ctx context.Context) error { - return nil - }, - s.storeConfig, - ) - if err != nil { - return errors.Wrap(err, "creating new store") - } - - newStore.isInitialized = s.isInitialized - - defer func() { - _ = tx.Rollback() - }() - - err = f(ctx, newStore) - if err != nil { - return errors.Wrap(err, "running transaction function") - } - - return sqlerrors.PostgresError(tx.Commit()) -} - -func (s *Store) instrumentalized(ctx context.Context, name string) func() { - now := time.Now() - attrs := []attribute.KeyValue{ - attribute.String("schema", s.schema.Name()), - attribute.String("op", name), - } - - return func() { - latency := time.Since(now) - s.metricsRegistry.Latencies.Record(ctx, latency.Milliseconds(), attrs...) - } -} - func (s *Store) OnLogWrote(fn OnLogWrote) { s.onLogsWrote = append(s.onLogsWrote, fn) } -func NewStore( - schema schema.Schema, +func New( + schema storage.Schema, onDelete func(ctx context.Context) error, storeConfig StoreConfig, ) (*Store, error) { - s := &Store{ + return &Store{ schema: schema, onDelete: onDelete, storeConfig: storeConfig, writeChannel: make(chan pendingLog, storeConfig.StoreWorkerConfig.MaxWriteChanSize), stopChan: make(chan chan struct{}, 1), - stopped: make(chan struct{}), - } - - metricsRegistry, err := metrics.RegisterSQLStorageMetrics(s.schema.Name()) - if err != nil { - return nil, errors.Wrap(err, "registering metrics") - } - s.metricsRegistry = metricsRegistry - - return s, nil + }, nil } diff --git a/components/ledger/pkg/storage/ledgerstore/store_test.go b/components/ledger/pkg/storage/ledgerstore/store_test.go index ec3f16ae79..5d991856f7 100644 --- a/components/ledger/pkg/storage/ledgerstore/store_test.go +++ b/components/ledger/pkg/storage/ledgerstore/store_test.go @@ -2,331 +2,14 @@ package ledgerstore_test import ( "context" - "math/big" "testing" - "time" "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/storage/errors" "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/stack/libs/go-libs/metadata" + "github.com/formancehq/stack/libs/go-libs/collectionutils" "github.com/stretchr/testify/require" ) -var now = core.Now() -var tx1 = core.ExpandedTransaction{ - Transaction: core.Transaction{ - TransactionData: core.TransactionData{ - Postings: []core.Posting{ - { - Source: "world", - Destination: "central_bank", - Amount: big.NewInt(100), - Asset: "USD", - }, - }, - Reference: "tx1", - Timestamp: now.Add(-3 * time.Hour), - }, - }, - PostCommitVolumes: core.AccountsAssetsVolumes{ - "world": { - "USD": { - Input: big.NewInt(0), - Output: big.NewInt(100), - }, - }, - "central_bank": { - "USD": { - Input: big.NewInt(100), - Output: big.NewInt(0), - }, - }, - }, - PreCommitVolumes: core.AccountsAssetsVolumes{ - "world": { - "USD": { - Input: big.NewInt(0), - Output: big.NewInt(0), - }, - }, - "central_bank": { - "USD": { - Input: big.NewInt(0), - Output: big.NewInt(0), - }, - }, - }, -} -var tx2 = core.ExpandedTransaction{ - Transaction: core.Transaction{ - ID: 1, - TransactionData: core.TransactionData{ - Postings: []core.Posting{ - { - Source: "world", - Destination: "central_bank", - Amount: big.NewInt(100), - Asset: "USD", - }, - }, - Reference: "tx2", - Timestamp: now.Add(-2 * time.Hour), - }, - }, - PostCommitVolumes: core.AccountsAssetsVolumes{ - "world": { - "USD": { - Input: big.NewInt(0), - Output: big.NewInt(200), - }, - }, - "central_bank": { - "USD": { - Input: big.NewInt(200), - Output: big.NewInt(0), - }, - }, - }, - PreCommitVolumes: core.AccountsAssetsVolumes{ - "world": { - "USD": { - Input: big.NewInt(0), - Output: big.NewInt(100), - }, - }, - "central_bank": { - "USD": { - Input: big.NewInt(100), - Output: big.NewInt(0), - }, - }, - }, -} -var tx3 = core.ExpandedTransaction{ - Transaction: core.Transaction{ - ID: 2, - TransactionData: core.TransactionData{ - Postings: []core.Posting{ - { - Source: "central_bank", - Destination: "users:1", - Amount: big.NewInt(1), - Asset: "USD", - }, - }, - Reference: "tx3", - Metadata: metadata.Metadata{ - "priority": "high", - }, - Timestamp: now.Add(-1 * time.Hour), - }, - }, - PreCommitVolumes: core.AccountsAssetsVolumes{ - "central_bank": { - "USD": { - Input: big.NewInt(200), - Output: big.NewInt(0), - }, - }, - "users:1": { - "USD": { - Input: big.NewInt(0), - Output: big.NewInt(0), - }, - }, - }, - PostCommitVolumes: core.AccountsAssetsVolumes{ - "central_bank": { - "USD": { - Input: big.NewInt(200), - Output: big.NewInt(1), - }, - }, - "users:1": { - "USD": { - Input: big.NewInt(1), - Output: big.NewInt(0), - }, - }, - }, -} - -func TestUpdateTransactionMetadata(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - tx := core.ExpandedTransaction{ - Transaction: core.Transaction{ - ID: 0, - TransactionData: core.TransactionData{ - Postings: []core.Posting{ - { - Source: "world", - Destination: "central_bank", - Amount: big.NewInt(100), - Asset: "USD", - }, - }, - Reference: "foo", - Timestamp: core.Now(), - }, - }, - } - err := store.InsertTransactions(context.Background(), tx) - require.NoError(t, err) - - err = store.UpdateTransactionMetadata(context.Background(), tx.ID, metadata.Metadata{ - "foo": "bar", - }) - require.NoError(t, err) - - retrievedTransaction, err := store.GetTransaction(context.Background(), tx.ID) - require.NoError(t, err) - require.EqualValues(t, "bar", retrievedTransaction.Metadata["foo"]) -} - -func TestUpdateAccountMetadata(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - require.NoError(t, store.EnsureAccountExists(context.Background(), "central_bank")) - - err := store.UpdateAccountMetadata(context.Background(), "central_bank", metadata.Metadata{ - "foo": "bar", - }) - require.NoError(t, err) - - account, err := store.GetAccount(context.Background(), "central_bank") - require.NoError(t, err) - require.EqualValues(t, "bar", account.Metadata["foo"]) -} - -func TestGetAccountNotFound(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - account, err := store.GetAccount(context.Background(), "account_not_existing") - require.True(t, errors.IsNotFoundError(err)) - require.Nil(t, account) -} - -func TestCountAccounts(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - require.NoError(t, store.EnsureAccountExists(context.Background(), "world")) - require.NoError(t, store.EnsureAccountExists(context.Background(), "central_bank")) - - countAccounts, err := store.CountAccounts(context.Background(), ledgerstore.AccountsQuery{}) - require.NoError(t, err) - require.EqualValues(t, 2, countAccounts) // world + central_bank -} - -func TestGetAssetsVolumes(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - require.NoError(t, store.UpdateVolumes(context.Background(), core.AccountsAssetsVolumes{ - "central_bank": { - "USD": { - Input: big.NewInt(100), - Output: big.NewInt(0), - }, - }, - })) - - volumes, err := store.GetAssetsVolumes(context.Background(), "central_bank") - require.NoError(t, err) - require.Len(t, volumes, 1) - require.EqualValues(t, big.NewInt(100), volumes["USD"].Input) - require.EqualValues(t, big.NewInt(0), volumes["USD"].Output) -} - -func TestGetAccounts(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - require.NoError(t, store.UpdateAccountMetadata(context.Background(), "world", metadata.Metadata{ - "foo": "bar", - })) - require.NoError(t, store.UpdateAccountMetadata(context.Background(), "bank", metadata.Metadata{ - "hello": "world", - })) - require.NoError(t, store.UpdateAccountMetadata(context.Background(), "order:1", metadata.Metadata{ - "hello": "world", - })) - require.NoError(t, store.UpdateAccountMetadata(context.Background(), "order:2", metadata.Metadata{ - "number": `3`, - "boolean": `true`, - "a": `{"super": {"nested": {"key": "hello"}}}`, - })) - - accounts, err := store.GetAccounts(context.Background(), - ledgerstore.NewAccountsQuery().WithPageSize(1), - ) - require.NoError(t, err) - require.Equal(t, 1, accounts.PageSize) - require.Len(t, accounts.Data, 1) - - accounts, err = store.GetAccounts(context.Background(), - ledgerstore.NewAccountsQuery(). - WithPageSize(1). - WithAfterAddress(accounts.Data[0].Address), - ) - require.NoError(t, err) - require.Equal(t, 1, accounts.PageSize) - - accounts, err = store.GetAccounts(context.Background(), - ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithAddressFilter("order:.*"), - ) - require.NoError(t, err) - require.Len(t, accounts.Data, 2) - require.Equal(t, 10, accounts.PageSize) - - accounts, err = store.GetAccounts(context.Background(), - ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithMetadataFilter(metadata.Metadata{ - "foo": "bar", - }), - ) - require.NoError(t, err) - require.Len(t, accounts.Data, 1) - - accounts, err = store.GetAccounts(context.Background(), ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithMetadataFilter(metadata.Metadata{ - "number": "3", - })) - require.NoError(t, err) - require.Len(t, accounts.Data, 1) - - accounts, err = store.GetAccounts(context.Background(), - ledgerstore.NewAccountsQuery(). - WithPageSize(10). - WithMetadataFilter(metadata.Metadata{ - "boolean": "true", - }), - ) - require.NoError(t, err) - require.Len(t, accounts.Data, 1) -} - -func TestGetTransaction(t *testing.T) { - t.Parallel() - store := newLedgerStore(t) - - require.NoError(t, store.InsertTransactions(context.Background(), tx1, tx2)) - - tx, err := store.GetTransaction(context.Background(), tx1.ID) - require.NoError(t, err) - require.Equal(t, tx1.Postings, tx.Postings) - require.Equal(t, tx1.Reference, tx.Reference) - require.Equal(t, tx1.Timestamp, tx.Timestamp) -} - func TestInitializeStore(t *testing.T) { t.Parallel() store := newLedgerStore(t) @@ -335,3 +18,14 @@ func TestInitializeStore(t *testing.T) { require.NoError(t, err) require.False(t, modified) } + +func insertTransactions(ctx context.Context, s *ledgerstore.Store, txs ...core.Transaction) error { + if err := s.InsertTransactions(ctx, txs...); err != nil { + return err + } + moves := collectionutils.Flatten(collectionutils.Map(txs, core.Transaction.GetMoves)) + if err := s.InsertMoves(ctx, moves...); err != nil { + return err + } + return nil +} diff --git a/components/ledger/pkg/storage/ledgerstore/transactions.go b/components/ledger/pkg/storage/ledgerstore/transactions.go index affebea92e..c548d3cc1a 100644 --- a/components/ledger/pkg/storage/ledgerstore/transactions.go +++ b/components/ledger/pkg/storage/ledgerstore/transactions.go @@ -2,57 +2,166 @@ package ledgerstore import ( "context" + "database/sql" "database/sql/driver" "encoding/json" "fmt" "math/big" + "sort" "strings" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/api" "github.com/formancehq/stack/libs/go-libs/metadata" - "github.com/pkg/errors" + "github.com/formancehq/stack/libs/go-libs/pointer" "github.com/uptrace/bun" - "github.com/uptrace/bun/extra/bunbig" ) const ( TransactionsTableName = "transactions" - PostingsTableName = "postings" + MovesTableName = "moves" ) + +type TransactionsQuery ColumnPaginatedQuery[TransactionsQueryFilters] + +func NewTransactionsQuery() TransactionsQuery { + return TransactionsQuery{ + PageSize: QueryDefaultPageSize, + Column: "id", + Order: OrderDesc, + Filters: TransactionsQueryFilters{ + Metadata: metadata.Metadata{}, + }, + } +} + +type TransactionsQueryFilters struct { + AfterTxID uint64 `json:"afterTxID,omitempty"` + Reference string `json:"reference,omitempty"` + Destination string `json:"destination,omitempty"` + Source string `json:"source,omitempty"` + Account string `json:"account,omitempty"` + EndTime core.Time `json:"endTime,omitempty"` + StartTime core.Time `json:"startTime,omitempty"` + Metadata metadata.Metadata `json:"metadata,omitempty"` +} + +func (a TransactionsQuery) WithPageSize(pageSize uint64) TransactionsQuery { + if pageSize != 0 { + a.PageSize = pageSize + } + + return a +} + +func (a TransactionsQuery) WithAfterTxID(after uint64) TransactionsQuery { + a.Filters.AfterTxID = after + + return a +} + +func (a TransactionsQuery) WithStartTimeFilter(start core.Time) TransactionsQuery { + if !start.IsZero() { + a.Filters.StartTime = start + } + + return a +} + +func (a TransactionsQuery) WithEndTimeFilter(end core.Time) TransactionsQuery { + if !end.IsZero() { + a.Filters.EndTime = end + } + + return a +} + +func (a TransactionsQuery) WithAccountFilter(account string) TransactionsQuery { + a.Filters.Account = account + + return a +} + +func (a TransactionsQuery) WithDestinationFilter(dest string) TransactionsQuery { + a.Filters.Destination = dest + + return a +} + +func (a TransactionsQuery) WithReferenceFilter(ref string) TransactionsQuery { + a.Filters.Reference = ref + + return a +} + +func (a TransactionsQuery) WithSourceFilter(source string) TransactionsQuery { + a.Filters.Source = source + + return a +} + +func (a TransactionsQuery) WithMetadataFilter(metadata metadata.Metadata) TransactionsQuery { + a.Filters.Metadata = metadata + + return a +} + type Transaction struct { bun.BaseModel `bun:"transactions,alias:transactions"` ID uint64 `bun:"id,type:bigint,pk"` Timestamp core.Time `bun:"timestamp,type:timestamptz"` Reference string `bun:"reference,type:varchar,unique,nullzero"` - Postings []Posting `bun:"rel:has-many,join:id=txid"` + Moves []Move `bun:"rel:has-many,join:id=transaction_id"` Metadata metadata.Metadata `bun:"metadata,type:jsonb,default:'{}'"` - //TODO(gfyrag): change to bytea - PreCommitVolumes core.AccountsAssetsVolumes `bun:"pre_commit_volumes,type:bytea"` - PostCommitVolumes core.AccountsAssetsVolumes `bun:"post_commit_volumes,type:bytea"` } func (t Transaction) toCore() core.ExpandedTransaction { - postings := core.Postings{} - for _, p := range t.Postings { - postings = append(postings, p.toCore()) - } - return core.ExpandedTransaction{ + //data, _ := json.MarshalIndent(t, "", " ") + //fmt.Println(string(data)) + ret := core.ExpandedTransaction{ Transaction: core.Transaction{ TransactionData: core.TransactionData{ - Postings: postings, Reference: t.Reference, Metadata: t.Metadata, Timestamp: t.Timestamp, + Postings: make(core.Postings, len(t.Moves)/2), }, ID: t.ID, }, - PreCommitVolumes: t.PreCommitVolumes, - PostCommitVolumes: t.PostCommitVolumes, + PreCommitVolumes: map[string]core.VolumesByAssets{}, + PostCommitVolumes: map[string]core.VolumesByAssets{}, + } + for _, m := range t.Moves { + ret.Postings[m.PostingIndex].Amount = (*big.Int)(m.Amount) + ret.Postings[m.PostingIndex].Asset = m.Asset + if m.IsSource { + ret.Postings[m.PostingIndex].Source = m.Account + } else { + ret.Postings[m.PostingIndex].Destination = m.Account + } + if _, ok := ret.PostCommitVolumes[m.Account]; !ok { + ret.PostCommitVolumes[m.Account] = map[string]*core.Volumes{} + ret.PreCommitVolumes[m.Account] = map[string]*core.Volumes{} + } + if _, ok := ret.PostCommitVolumes[m.Account][m.Asset]; !ok { + ret.PostCommitVolumes[m.Account][m.Asset] = core.NewEmptyVolumes() + ret.PreCommitVolumes[m.Account][m.Asset] = core.NewEmptyVolumes() + } + + ret.PostCommitVolumes[m.Account][m.Asset].Output = NewInt().Set(m.PostCommitOutputVolume).ToMathBig() + ret.PostCommitVolumes[m.Account][m.Asset].Input = NewInt().Set(m.PostCommitInputVolume).ToMathBig() + if m.IsSource { + ret.PreCommitVolumes[m.Account][m.Asset].Output = NewInt().Sub(m.PostCommitOutputVolume, m.Amount).ToMathBig() + ret.PreCommitVolumes[m.Account][m.Asset].Input = NewInt().Set(m.PostCommitInputVolume).ToMathBig() + } else { + ret.PreCommitVolumes[m.Account][m.Asset].Output = NewInt().Set(m.PostCommitOutputVolume).ToMathBig() + ret.PreCommitVolumes[m.Account][m.Asset].Input = NewInt().Sub(m.PostCommitInputVolume, m.Amount).ToMathBig() + } } + return ret } type account string @@ -93,106 +202,95 @@ func (m1 *account) Scan(value interface{}) error { return nil } -type Posting struct { - bun.BaseModel `bun:"postings,alias:postings"` - - Transaction *Transaction `bun:"rel:belongs-to,join:txid=id"` - TransactionID uint64 `bun:"txid,type:bigint"` - Amount *bunbig.Int `bun:"amount,type:bigint"` - Asset string `bun:"asset,type:string"` - Source account `bun:"source,type:jsonb"` - Destination account `bun:"destination,type:jsonb"` - Index uint8 `bun:"index,type:int8"` -} - -func (p Posting) toCore() core.Posting { - return core.Posting{ - Source: string(p.Source), - Destination: string(p.Destination), - Amount: (*big.Int)(p.Amount), - Asset: p.Asset, - } +type Move struct { + bun.BaseModel `bun:"moves,alias:m"` + + TransactionID uint64 `bun:"transaction_id,type:bigint" json:"transaction_id"` + Amount *Int `bun:"amount,type:bigint" json:"amount"` + Asset string `bun:"asset,type:varchar" json:"asset"` + Account string `bun:"account,type:varchar" json:"account"` + AccountArray []string `bun:"account_array,type:jsonb" json:"account_array"` + PostingIndex uint8 `bun:"posting_index,type:int8" json:"posting_index"` + IsSource bool `bun:"is_source,type:bool" json:"is_source"` + Timestamp core.Time `bun:"timestamp,type:timestamp" json:"timestamp"` + PostCommitInputVolume *Int `bun:"post_commit_input_value,type:numeric" json:"post_commit_input_value"` + PostCommitOutputVolume *Int `bun:"post_commit_output_value,type:numeric" json:"post_commit_output_value"` } func (s *Store) buildTransactionsQuery(p TransactionsQueryFilters, models *[]Transaction) *bun.SelectQuery { - sb := s.schema. - NewSelect(TransactionsTableName). - Model(models). - Relation("Postings", func(sb *bun.SelectQuery) *bun.SelectQuery { - return sb.With("postings", s.schema.NewSelect(PostingsTableName)) - }). - Distinct() - - if p.Source != "" || p.Destination != "" || p.Account != "" { - sb = sb. - Join(fmt.Sprintf("JOIN %s", s.schema.Table(PostingsTableName))). - JoinOn("postings.txid = transactions.id") - } - if p.Source != "" { - src := strings.Split(p.Source, ":") - sb.Where(fmt.Sprintf("jsonb_array_length(postings.source) = %d", len(src))) - - for i, segment := range src { - if segment == ".*" || segment == "*" || segment == "" { - continue - } - - sb.Where(fmt.Sprintf("postings.source @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) - } - } - if p.Destination != "" { - dst := strings.Split(p.Destination, ":") - sb.Where(fmt.Sprintf("jsonb_array_length(postings.destination) = %d", len(dst))) - for i, segment := range dst { - if segment == ".*" || segment == "*" || segment == "" { - continue - } - - sb.Where(fmt.Sprintf("postings.destination @@ ('$[%d] == \"' || ?::text || '\"')::jsonpath", i), segment) - } - } - if p.Account != "" { - dst := strings.Split(p.Account, ":") - sb.Where(fmt.Sprintf("(jsonb_array_length(postings.destination) = %d OR jsonb_array_length(postings.source) = %d)", len(dst), len(dst))) - for i, segment := range dst { - if segment == ".*" || segment == "*" || segment == "" { - continue - } - - sb.Where(fmt.Sprintf("(postings.source @@ ('$[%d] == \"' || ?0::text || '\"')::jsonpath OR postings.destination @@ ('$[%d] == \"' || ?0::text || '\"')::jsonpath)", i, i), segment) - } - } + selectMatchingTransactions := s.schema.NewSelect(TransactionsTableName). + ColumnExpr("distinct on(transactions.id) transactions.id as transaction_id") if p.Reference != "" { - sb.Where("reference = ?", p.Reference) + selectMatchingTransactions.Where("transactions.reference = ?", p.Reference) } if !p.StartTime.IsZero() { - sb.Where("timestamp >= ?", p.StartTime.UTC()) + selectMatchingTransactions.Where("transactions.timestamp >= ?", p.StartTime) } if !p.EndTime.IsZero() { - sb.Where("timestamp < ?", p.EndTime.UTC()) + selectMatchingTransactions.Where("transactions.timestamp < ?", p.EndTime) } - if p.AfterTxID > 0 { - sb.Where("id > ?", p.AfterTxID) + if p.AfterTxID != 0 { + selectMatchingTransactions.Where("transactions.id > ?", p.AfterTxID) } - - for key, value := range p.Metadata { - sb.Where(s.schema.Table( - fmt.Sprintf("%s(metadata, ?, '%s')", - SQLCustomFuncMetaCompare, strings.ReplaceAll(key, ".", "', '")), - ), value) + if p.Metadata != nil && len(p.Metadata) > 0 { + selectMatchingTransactions.Where("transactions.metadata @> ?", p.Metadata) + } + if p.Source != "" || p.Destination != "" || p.Account != "" { + selectMatchingTransactions.Join(fmt.Sprintf("join %s m on transactions.id = m.transaction_id", s.schema.Table("moves"))) + if p.Source != "" { + parts := strings.Split(p.Source, ":") + selectMatchingTransactions.Where(fmt.Sprintf("m.is_source and jsonb_array_length(m.account_array) = %d", len(parts))) + for index, segment := range parts { + if len(segment) == 0 { + continue + } + selectMatchingTransactions.Where(fmt.Sprintf(`m.account_array @@ ('$[%d] == "%s"')`, index, segment)) + } + } + if p.Destination != "" { + parts := strings.Split(p.Destination, ":") + selectMatchingTransactions.Where(fmt.Sprintf("not m.is_source and jsonb_array_length(m.account_array) = %d", len(parts))) + for index, segment := range parts { + if len(segment) == 0 { + continue + } + selectMatchingTransactions.Where(fmt.Sprintf(`m.account_array @@ ('$[%d] == "%s"')`, index, segment)) + } + } + if p.Account != "" { + parts := strings.Split(p.Account, ":") + selectMatchingTransactions.Where(fmt.Sprintf("jsonb_array_length(m.account_array) = %d", len(parts))) + for index, segment := range parts { + if len(segment) == 0 { + continue + } + selectMatchingTransactions.Where(fmt.Sprintf(`m.account_array @@ ('$[%d] == "%s"')`, index, segment)) + } + } } - return sb + return s.schema.NewSelect(TransactionsTableName). + Model(models). + Column("transactions.id", "transactions.reference", "transactions.metadata", "transactions.timestamp"). + ColumnExpr(`json_agg(json_build_object( + 'posting_index', m.posting_index, + 'transaction_id', m.transaction_id, + 'account', m.account, + 'account_array', m.account_array, + 'asset', m.asset, + 'post_commit_input_value', m.post_commit_input_value, + 'post_commit_output_value', m.post_commit_output_value, + 'timestamp', m.timestamp, + 'amount', m.amount, + 'is_source', m.is_source + )) as moves`). + Join(fmt.Sprintf("join %s m on transactions.id = m.transaction_id", s.schema.Table("moves"))). + Join(fmt.Sprintf(`join (%s) ids on ids.transaction_id = transactions.id`, selectMatchingTransactions.String())). + Group("transactions.id") } func (s *Store) GetTransactions(ctx context.Context, q TransactionsQuery) (*api.Cursor[core.ExpandedTransaction], error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_transactions") - defer recordMetrics() - cursor, err := UsingColumn[TransactionsQueryFilters, Transaction](ctx, s.buildTransactionsQuery, ColumnPaginatedQuery[TransactionsQueryFilters](q), ) @@ -204,12 +302,6 @@ func (s *Store) GetTransactions(ctx context.Context, q TransactionsQuery) (*api. } func (s *Store) CountTransactions(ctx context.Context, q TransactionsQuery) (uint64, error) { - if !s.isInitialized { - return 0, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "count_transactions") - defer recordMetrics() - models := make([]Transaction, 0) count, err := s.buildTransactionsQuery(q.Filters, &models).Count(ctx) @@ -217,17 +309,11 @@ func (s *Store) CountTransactions(ctx context.Context, q TransactionsQuery) (uin } func (s *Store) GetTransaction(ctx context.Context, txId uint64) (*core.ExpandedTransaction, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_transaction") - defer recordMetrics() - tx := &Transaction{} err := s.schema.NewSelect(TransactionsTableName). Model(tx). - Relation("Postings", func(query *bun.SelectQuery) *bun.SelectQuery { - return query.With("postings", s.schema.NewSelect(PostingsTableName)) + Relation("Moves", func(query *bun.SelectQuery) *bun.SelectQuery { + return query.With("moves", s.schema.NewSelect(MovesTableName)) }). Where("id = ?", txId). OrderExpr("id DESC"). @@ -235,95 +321,217 @@ func (s *Store) GetTransaction(ctx context.Context, txId uint64) (*core.Expanded if err != nil { return nil, storageerrors.PostgresError(err) } - coreTx := tx.toCore() - return &coreTx, nil + return pointer.For(tx.toCore()), nil } -func (s *Store) insertTransactions(ctx context.Context, txs ...core.ExpandedTransaction) error { +func (s *Store) InsertTransactions(ctx context.Context, txs ...core.Transaction) error { + ts := make([]Transaction, len(txs)) - ps := make([]Posting, 0) for i, tx := range txs { ts[i].ID = tx.ID ts[i].Timestamp = tx.Timestamp ts[i].Metadata = tx.Metadata - ts[i].PreCommitVolumes = tx.PreCommitVolumes - ts[i].PostCommitVolumes = tx.PostCommitVolumes ts[i].Reference = "" if tx.Reference != "" { cp := tx.Reference ts[i].Reference = cp } - - for i, p := range tx.Postings { - ps = append(ps, Posting{ - TransactionID: tx.ID, - Amount: (*bunbig.Int)(p.Amount), - Asset: p.Asset, - Source: account(p.Source), - Destination: account(p.Destination), - Index: uint8(i), - }) - } } _, err := s.schema.NewInsert(TransactionsTableName). Model(&ts). On("CONFLICT (id) DO NOTHING"). Exec(ctx) - if err != nil { - return storageerrors.PostgresError(err) - } - - _, err = s.schema.NewInsert(PostingsTableName). - Model(&ps). - On("CONFLICT (txid, index) DO NOTHING"). - Exec(ctx) return storageerrors.PostgresError(err) } -func (s *Store) InsertTransactions(ctx context.Context, txs ...core.ExpandedTransaction) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "insert_transactions") - defer recordMetrics() +func (s *Store) InsertMoves(ctx context.Context, objects ...*core.Move) error { + type moveValue struct { + Move + AmountInputBefore *big.Int `bun:"amount_input_before,type:numeric"` + AmountOutputBefore *big.Int `bun:"amount_output_before,type:numeric"` + AccumulatedPostingAmount *big.Int `bun:"accumulated_posting_amount,type:numeric"` + } + + transactionIds := make([]uint64, 0) + moves := make([]moveValue, 0) + + sort.Slice(objects, func(i, j int) bool { + if objects[i].Timestamp.Equal(objects[j].Timestamp) { + if objects[i].TransactionID == objects[j].TransactionID { + if objects[i].PostingIndex == objects[j].PostingIndex { + if objects[i].IsSource { + return false + } + } + return objects[i].PostingIndex < objects[j].PostingIndex + } + return objects[i].TransactionID < objects[j].TransactionID + } + return objects[i].Timestamp.Before(objects[j].Timestamp) + }) - return storageerrors.PostgresError(s.insertTransactions(ctx, txs...)) -} + var ( + accumulatedAmounts core.AccountsAssetsVolumes + actualTransactionID *uint64 + actualAccumulatedVolumesOnTransaction core.AccountsAssetsVolumes + ) + + for i := 0; i < len(objects); { + + if actualTransactionID == nil || objects[i].TransactionID != *actualTransactionID { + actualTransactionID = &objects[i].TransactionID + actualAccumulatedVolumesOnTransaction = core.AccountsAssetsVolumes{} + transactionIds = append(transactionIds, *actualTransactionID) + } + + for j := i; j < len(objects) && objects[j].TransactionID == *actualTransactionID; j++ { + if objects[j].IsSource { + actualAccumulatedVolumesOnTransaction.AddOutput(objects[j].Account, objects[j].Asset, objects[j].Amount) + } else { + actualAccumulatedVolumesOnTransaction.AddInput(objects[j].Account, objects[j].Asset, objects[j].Amount) + } + } + + j := i + for ; j < len(objects) && objects[j].TransactionID == *actualTransactionID; j++ { + if objects[j].IsSource { + moves = append(moves, moveValue{ + Move: Move{ + TransactionID: *actualTransactionID, + Amount: (*Int)(objects[j].Amount), + Asset: objects[j].Asset, + Account: objects[j].Account, + AccountArray: strings.Split(objects[j].Account, ":"), + PostingIndex: objects[j].PostingIndex, + IsSource: true, + Timestamp: objects[j].Timestamp, + }, + AmountOutputBefore: accumulatedAmounts.GetVolumes(objects[j].Account, objects[j].Asset).Output, + AmountInputBefore: accumulatedAmounts.GetVolumes(objects[j].Account, objects[j].Asset).Input, + AccumulatedPostingAmount: actualAccumulatedVolumesOnTransaction.GetVolumes(objects[j].Account, objects[j].Asset).Output, + }) + } else { + moves = append(moves, moveValue{ + Move: Move{ + TransactionID: *actualTransactionID, + Amount: (*Int)(objects[j].Amount), + Asset: objects[j].Asset, + Account: objects[j].Account, + AccountArray: strings.Split(objects[j].Account, ":"), + PostingIndex: objects[j].PostingIndex, + IsSource: false, + Timestamp: objects[j].Timestamp, + }, + AmountOutputBefore: accumulatedAmounts.GetVolumes(objects[j].Account, objects[j].Asset).Output, + AmountInputBefore: accumulatedAmounts.GetVolumes(objects[j].Account, objects[j].Asset).Input, + AccumulatedPostingAmount: actualAccumulatedVolumesOnTransaction.GetVolumes(objects[j].Account, objects[j].Asset).Input, + }) + } + + if objects[j].IsSource { + accumulatedAmounts.AddOutput(objects[j].Account, objects[j].Asset, objects[j].Amount) + } else { + accumulatedAmounts.AddInput(objects[j].Account, objects[j].Asset, objects[j].Amount) + } + } + + i = j + } -func (s *Store) UpdateTransactionMetadata(ctx context.Context, id uint64, metadata metadata.Metadata) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) + type insertedMove struct { + TransactionID uint64 `bun:"transaction_id"` + PostingIndex uint8 `bun:"posting_index"` + IsSource bool `bun:"is_source"` } - recordMetrics := s.instrumentalized(ctx, "update_transaction_metadata") - defer recordMetrics() + insertedMoves := make([]insertedMove, 0) - metadataData, err := json.Marshal(metadata) + tx, err := s.schema.BeginTx(ctx, &sql.TxOptions{}) + if err != nil { + return err + } + defer func() { + _ = tx.Rollback() + }() + + err = tx.NewInsert(MovesTableName). + With("cte1", s.schema.NewValues(&moves)). + Column( + "posting_index", + "transaction_id", + "account", + "post_commit_input_value", + "post_commit_output_value", + "timestamp", + "asset", + "account_array", + "amount", + "is_source", + ). + TableExpr(fmt.Sprintf(` + (select cte1.posting_index, cte1.transaction_id::numeric, cte1.account, coalesce( + (select post_commit_input_value + from %s + where account = cte1.account and asset = cte1.asset and timestamp <= cte1.timestamp + order by timestamp desc, transaction_id desc + limit 1) + , 0) + cte1.amount_input_before + (case when not cte1.is_source then cte1.accumulated_posting_amount else 0 end) as post_commit_input_value, coalesce( + (select post_commit_output_value + from %s + where account = cte1.account and asset = cte1.asset and timestamp <= cte1.timestamp + order by timestamp desc, transaction_id desc + limit 1) + , 0) + cte1.amount_output_before + (case when cte1.is_source then cte1.accumulated_posting_amount else 0 end) as post_commit_output_value, cte1.timestamp, cte1.asset, cte1.account_array, cte1.amount, cte1.is_source + from cte1) data + `, s.schema.Table(MovesTableName), s.schema.Table(MovesTableName))). + On("CONFLICT DO NOTHING"). + Returning("transaction_id, posting_index, is_source"). + Scan(ctx, &insertedMoves) if err != nil { - return errors.Wrap(err, "failed to marshal metadata") + return storageerrors.PostgresError(err) + } + if len(insertedMoves) != len(moves) { // Some conflict (maybe after a crash?), we need to filter already inserted moves + ind := 0 + l: + for _, move := range moves { + for _, insertedMove := range insertedMoves { + if move.TransactionID == insertedMove.TransactionID && + move.PostingIndex == insertedMove.PostingIndex && + move.IsSource == insertedMove.IsSource { + ind++ + continue l + } + } + if ind < len(moves)-1 { + moves = append(moves[:ind], moves[ind+1:]...) + } else { + moves = moves[:ind] + } + } } - _, err = s.schema.NewUpdate(TransactionsTableName). - Model((*Transaction)(nil)). - Set("metadata = metadata || ?", string(metadataData)). - Where("id = ?", id). - Exec(ctx) + if len(moves) > 0 { + _, err = tx.NewUpdate(MovesTableName). + With("cte1", s.schema.NewValues(&moves)). + Set("post_commit_output_value = moves.post_commit_output_value + (case when cte1.is_source then cte1.amount else 0 end)"). + Set("post_commit_input_value = moves.post_commit_input_value + (case when not cte1.is_source then cte1.amount else 0 end)"). + Table("cte1"). + Where("moves.timestamp > cte1.timestamp and moves.account = cte1.account and moves.asset = cte1.asset and moves.transaction_id not in (?)", bun.In(transactionIds)). + Exec(ctx) + if err != nil { + return storageerrors.PostgresError(err) + } + } - return storageerrors.PostgresError(err) + return storageerrors.PostgresError(tx.Commit()) } func (s *Store) UpdateTransactionsMetadata(ctx context.Context, transactionsWithMetadata ...core.TransactionWithMetadata) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "update_transactions_metadata") - defer recordMetrics() - txs := make([]*Transaction, 0, len(transactionsWithMetadata)) for _, tx := range transactionsWithMetadata { txs = append(txs, &Transaction{ @@ -332,10 +540,8 @@ func (s *Store) UpdateTransactionsMetadata(ctx context.Context, transactionsWith }) } - values := s.schema.NewValues(&txs) - _, err := s.schema.NewUpdate(TransactionsTableName). - With("_data", values). + With("_data", s.schema.NewValues(&txs)). Model((*Transaction)(nil)). TableExpr("_data"). Set("metadata = transactions.metadata || _data.metadata"). @@ -344,87 +550,3 @@ func (s *Store) UpdateTransactionsMetadata(ctx context.Context, transactionsWith return storageerrors.PostgresError(err) } - -type TransactionsQuery ColumnPaginatedQuery[TransactionsQueryFilters] - -func NewTransactionsQuery() TransactionsQuery { - return TransactionsQuery{ - PageSize: QueryDefaultPageSize, - Column: "id", - Order: OrderDesc, - Filters: TransactionsQueryFilters{ - Metadata: metadata.Metadata{}, - }, - } -} - -type TransactionsQueryFilters struct { - AfterTxID uint64 `json:"afterTxID,omitempty"` - Reference string `json:"reference,omitempty"` - Destination string `json:"destination,omitempty"` - Source string `json:"source,omitempty"` - Account string `json:"account,omitempty"` - EndTime core.Time `json:"endTime,omitempty"` - StartTime core.Time `json:"startTime,omitempty"` - Metadata metadata.Metadata `json:"metadata,omitempty"` -} - -func (a TransactionsQuery) WithPageSize(pageSize uint64) TransactionsQuery { - if pageSize != 0 { - a.PageSize = pageSize - } - - return a -} - -func (a TransactionsQuery) WithAfterTxID(after uint64) TransactionsQuery { - a.Filters.AfterTxID = after - - return a -} - -func (a TransactionsQuery) WithStartTimeFilter(start core.Time) TransactionsQuery { - if !start.IsZero() { - a.Filters.StartTime = start - } - - return a -} - -func (a TransactionsQuery) WithEndTimeFilter(end core.Time) TransactionsQuery { - if !end.IsZero() { - a.Filters.EndTime = end - } - - return a -} - -func (a TransactionsQuery) WithAccountFilter(account string) TransactionsQuery { - a.Filters.Account = account - - return a -} - -func (a TransactionsQuery) WithDestinationFilter(dest string) TransactionsQuery { - a.Filters.Destination = dest - - return a -} - -func (a TransactionsQuery) WithReferenceFilter(ref string) TransactionsQuery { - a.Filters.Reference = ref - - return a -} - -func (a TransactionsQuery) WithSourceFilter(source string) TransactionsQuery { - a.Filters.Source = source - - return a -} - -func (a TransactionsQuery) WithMetadataFilter(metadata metadata.Metadata) TransactionsQuery { - a.Filters.Metadata = metadata - - return a -} diff --git a/components/ledger/pkg/storage/ledgerstore/transactions_test.go b/components/ledger/pkg/storage/ledgerstore/transactions_test.go index d09963c0d1..86eb95a0cc 100644 --- a/components/ledger/pkg/storage/ledgerstore/transactions_test.go +++ b/components/ledger/pkg/storage/ledgerstore/transactions_test.go @@ -10,11 +10,157 @@ import ( "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/api" "github.com/formancehq/stack/libs/go-libs/metadata" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/require" ) -func TestTransactions(t *testing.T) { +func bigIntComparer(v1 *big.Int, v2 *big.Int) bool { + return v1.String() == v2.String() +} + +func RequireEqual(t *testing.T, expected, actual any) { + t.Helper() + if diff := cmp.Diff(expected, actual, cmp.Comparer(bigIntComparer)); diff != "" { + require.Failf(t, "Content not matching", diff) + } +} + +func ExpandTransactions(txs ...*core.Transaction) []core.ExpandedTransaction { + ret := make([]core.ExpandedTransaction, len(txs)) + accumulatedVolumes := core.AccountsAssetsVolumes{} + for ind, tx := range txs { + ret[ind].Transaction = *tx + for _, posting := range tx.Postings { + ret[ind].PreCommitVolumes.AddInput(posting.Destination, posting.Asset, accumulatedVolumes.GetVolumes(posting.Destination, posting.Asset).Input) + ret[ind].PreCommitVolumes.AddOutput(posting.Source, posting.Asset, accumulatedVolumes.GetVolumes(posting.Source, posting.Asset).Output) + } + for _, posting := range tx.Postings { + accumulatedVolumes.AddOutput(posting.Source, posting.Asset, posting.Amount) + accumulatedVolumes.AddInput(posting.Destination, posting.Asset, posting.Amount) + } + for _, posting := range tx.Postings { + ret[ind].PostCommitVolumes.AddInput(posting.Destination, posting.Asset, accumulatedVolumes.GetVolumes(posting.Destination, posting.Asset).Input) + ret[ind].PostCommitVolumes.AddOutput(posting.Source, posting.Asset, accumulatedVolumes.GetVolumes(posting.Source, posting.Asset).Output) + } + } + return ret +} + +func Reverse[T any](values ...T) []T { + for i := 0; i < len(values)/2; i++ { + values[i], values[len(values)-i-1] = values[len(values)-i-1], values[i] + } + return values +} + +func TestGetTransaction(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() + + tx1 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + TransactionData: core.TransactionData{ + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: big.NewInt(100), + Asset: "USD", + }, + }, + Reference: "tx1", + Timestamp: now.Add(-3 * time.Hour), + }, + }, + PostCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(100), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(100), + Output: big.NewInt(0), + }, + }, + }, + PreCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(0), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(0), + }, + }, + }, + } + tx2 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 1, + TransactionData: core.TransactionData{ + Postings: []core.Posting{ + { + Source: "world", + Destination: "central_bank", + Amount: big.NewInt(100), + Asset: "USD", + }, + }, + Reference: "tx2", + Timestamp: now.Add(-2 * time.Hour), + }, + }, + PostCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(200), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(200), + Output: big.NewInt(0), + }, + }, + }, + PreCommitVolumes: core.AccountsAssetsVolumes{ + "world": { + "USD": { + Input: big.NewInt(0), + Output: big.NewInt(100), + }, + }, + "central_bank": { + "USD": { + Input: big.NewInt(100), + Output: big.NewInt(0), + }, + }, + }, + } + + require.NoError(t, insertTransactions(context.Background(), store, tx1.Transaction, tx2.Transaction)) + + tx, err := store.GetTransaction(context.Background(), tx1.ID) + require.NoError(t, err) + require.Equal(t, tx1.Postings, tx.Postings) + require.Equal(t, tx1.Reference, tx.Reference) + require.Equal(t, tx1.Timestamp, tx.Timestamp) +} + +func TestInsertTransactions(t *testing.T) { + t.Parallel() store := newLedgerStore(t) + now := core.Now() t.Run("success inserting transaction", func(t *testing.T) { tx1 := core.ExpandedTransaction{ @@ -33,14 +179,29 @@ func TestTransactions(t *testing.T) { Metadata: metadata.Metadata{}, }, }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + "alice": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(100), + }, + "alice": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(100), + }, + }, } - err := store.InsertTransactions(context.Background(), tx1) + err := insertTransactions(context.Background(), store, tx1.Transaction) require.NoError(t, err, "inserting transaction should not fail") tx, err := store.GetTransaction(context.Background(), 0) - require.NoError(t, err, "getting transaction should not fail") - require.Equal(t, &tx1, tx, "transaction should be equal") + RequireEqual(t, tx1, *tx) }) t.Run("success inserting multiple transactions", func(t *testing.T) { @@ -60,6 +221,22 @@ func TestTransactions(t *testing.T) { Metadata: metadata.Metadata{}, }, }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(100), + }, + "polo": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(300), + }, + "polo": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(200), + }, + }, } tx3 := core.ExpandedTransaction{ @@ -78,91 +255,458 @@ func TestTransactions(t *testing.T) { Metadata: metadata.Metadata{}, }, }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(300), + }, + "gfyrag": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(450), + }, + "gfyrag": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(150), + }, + }, } - err := store.InsertTransactions(context.Background(), tx2, tx3) + err := insertTransactions(context.Background(), store, tx2.Transaction, tx3.Transaction) require.NoError(t, err, "inserting multiple transactions should not fail") tx, err := store.GetTransaction(context.Background(), 1) require.NoError(t, err, "getting transaction should not fail") - require.Equal(t, &tx2, tx, "transaction should be equal") + RequireEqual(t, tx2, *tx) tx, err = store.GetTransaction(context.Background(), 2) require.NoError(t, err, "getting transaction should not fail") - require.Equal(t, &tx3, tx, "transaction should be equal") + RequireEqual(t, tx3, *tx) }) +} - t.Run("success counting transactions", func(t *testing.T) { - count, err := store.CountTransactions(context.Background(), ledgerstore.TransactionsQuery{}) - require.NoError(t, err, "counting transactions should not fail") - require.Equal(t, uint64(3), count, "count should be equal") - }) +func TestCountTransactions(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() - t.Run("success updating transaction metadata", func(t *testing.T) { - metadata := metadata.Metadata{ - "foo": "bar", - } - err := store.UpdateTransactionMetadata(context.Background(), 0, metadata) - require.NoError(t, err, "updating transaction metadata should not fail") + tx1 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 0, + TransactionData: core.TransactionData{ + Postings: core.Postings{ + { + Source: "world", + Destination: "alice", + Amount: big.NewInt(100), + Asset: "USD", + }, + }, + Timestamp: now.Add(-3 * time.Hour), + Metadata: metadata.Metadata{}, + }, + }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + "alice": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(100), + }, + "alice": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(100), + }, + }, + } + tx2 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 1, + TransactionData: core.TransactionData{ + Postings: core.Postings{ + { + Source: "world", + Destination: "polo", + Amount: big.NewInt(200), + Asset: "USD", + }, + }, + Timestamp: now.Add(-2 * time.Hour), + Metadata: metadata.Metadata{}, + }, + }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(100), + }, + "polo": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(300), + }, + "polo": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(200), + }, + }, + } - tx, err := store.GetTransaction(context.Background(), 0) - require.NoError(t, err, "getting transaction should not fail") - require.Equal(t, tx.Metadata, metadata, "metadata should be equal") - }) + tx3 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 2, + TransactionData: core.TransactionData{ + Postings: core.Postings{ + { + Source: "world", + Destination: "gfyrag", + Amount: big.NewInt(150), + Asset: "USD", + }, + }, + Timestamp: now.Add(-1 * time.Hour), + Metadata: metadata.Metadata{}, + }, + }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(300), + }, + "gfyrag": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(450), + }, + "gfyrag": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(150), + }, + }, + } - t.Run("success updating multiple transaction metadata", func(t *testing.T) { - txToUpdate1 := core.TransactionWithMetadata{ - ID: 1, - Metadata: metadata.Metadata{"foo1": "bar2"}, - } - txToUpdate2 := core.TransactionWithMetadata{ - ID: 2, - Metadata: metadata.Metadata{"foo2": "bar2"}, - } - txs := []core.TransactionWithMetadata{txToUpdate1, txToUpdate2} + err := insertTransactions(context.Background(), store, tx1.Transaction, tx2.Transaction, tx3.Transaction) + require.NoError(t, err, "inserting transaction should not fail") - err := store.UpdateTransactionsMetadata(context.Background(), txs...) - require.NoError(t, err, "updating multiple transaction metadata should not fail") + count, err := store.CountTransactions(context.Background(), ledgerstore.TransactionsQuery{}) + require.NoError(t, err, "counting transactions should not fail") + require.Equal(t, uint64(3), count, "count should be equal") +} - tx, err := store.GetTransaction(context.Background(), 1) - require.NoError(t, err, "getting transaction should not fail") - require.Equal(t, tx.Metadata, txToUpdate1.Metadata, "metadata should be equal") +func TestUpdateTransactionsMetadata(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() - tx, err = store.GetTransaction(context.Background(), 2) - require.NoError(t, err, "getting transaction should not fail") - require.Equal(t, tx.Metadata, txToUpdate2.Metadata, "metadata should be equal") - }) + tx1 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 0, + TransactionData: core.TransactionData{ + Postings: core.Postings{ + { + Source: "world", + Destination: "alice", + Amount: big.NewInt(100), + Asset: "USD", + }, + }, + Timestamp: now.Add(-3 * time.Hour), + Metadata: metadata.Metadata{}, + }, + }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + "alice": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(100), + }, + "alice": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(100), + }, + }, + } + tx2 := core.ExpandedTransaction{ + Transaction: core.Transaction{ + ID: 1, + TransactionData: core.TransactionData{ + Postings: core.Postings{ + { + Source: "world", + Destination: "polo", + Amount: big.NewInt(200), + Asset: "USD", + }, + }, + Timestamp: now.Add(-2 * time.Hour), + Metadata: metadata.Metadata{}, + }, + }, + PreCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(100), + }, + "polo": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes(), + }, + }, + PostCommitVolumes: map[string]core.VolumesByAssets{ + "world": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithOutputInt64(300), + }, + "polo": map[string]*core.Volumes{ + "USD": core.NewEmptyVolumes().WithInputInt64(200), + }, + }, + } + + err := insertTransactions(context.Background(), store, tx1.Transaction, tx2.Transaction) + require.NoError(t, err, "inserting transaction should not fail") + + txToUpdate1 := core.TransactionWithMetadata{ + ID: 0, + Metadata: metadata.Metadata{"foo1": "bar2"}, + } + txToUpdate2 := core.TransactionWithMetadata{ + ID: 1, + Metadata: metadata.Metadata{"foo2": "bar2"}, + } + txs := []core.TransactionWithMetadata{txToUpdate1, txToUpdate2} + + err = store.UpdateTransactionsMetadata(context.Background(), txs...) + require.NoError(t, err, "updating multiple transaction metadata should not fail") + + tx, err := store.GetTransaction(context.Background(), 0) + require.NoError(t, err, "getting transaction should not fail") + require.Equal(t, tx.Metadata, txToUpdate1.Metadata, "metadata should be equal") + + tx, err = store.GetTransaction(context.Background(), 1) + require.NoError(t, err, "getting transaction should not fail") + require.Equal(t, tx.Metadata, txToUpdate2.Metadata, "metadata should be equal") +} + +func TestInsertTransactionInPast(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() + + tx1 := core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), + ).WithTimestamp(now) + + tx2 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user1", "USD/2", big.NewInt(50)), + ).WithTimestamp(now.Add(time.Hour)).WithID(1) + + // Insert in past must modify pre/post commit volumes of tx2 + tx3 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user2", "USD/2", big.NewInt(50)), + ).WithTimestamp(now.Add(30 * time.Minute)).WithID(2) + + require.NoError(t, insertTransactions(context.Background(), store, *tx1, *tx2)) + require.NoError(t, insertTransactions(context.Background(), store, *tx3)) + + tx2FromDatabase, err := store.GetTransaction(context.Background(), tx2.ID) + require.NoError(t, err) + + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 50), + }, + "user1": { + "USD/2": core.NewVolumesInt64(0, 0), + }, + }, tx2FromDatabase.PreCommitVolumes) + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 100), + }, + "user1": { + "USD/2": core.NewVolumesInt64(50, 0), + }, + }, tx2FromDatabase.PostCommitVolumes) +} + +func TestInsertTransactionInPastInOneBatch(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() + + tx1 := core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), + ).WithTimestamp(now) + + tx2 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user1", "USD/2", big.NewInt(50)), + ).WithTimestamp(now.Add(time.Hour)).WithID(1) + + // Insert in past must modify pre/post commit volumes of tx2 + tx3 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user2", "USD/2", big.NewInt(50)), + ).WithTimestamp(now.Add(30 * time.Minute)).WithID(2) + + require.NoError(t, insertTransactions(context.Background(), store, *tx1, *tx2, *tx3)) + + tx2FromDatabase, err := store.GetTransaction(context.Background(), tx2.ID) + require.NoError(t, err) + + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 50), + }, + "user1": { + "USD/2": core.NewVolumesInt64(0, 0), + }, + }, tx2FromDatabase.PreCommitVolumes) + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 100), + }, + "user1": { + "USD/2": core.NewVolumesInt64(50, 0), + }, + }, tx2FromDatabase.PostCommitVolumes) +} + +func TestInsertTwoTransactionAtSameDateInSameBatch(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() + + tx1 := core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), + ).WithTimestamp(now.Add(-time.Hour)) + + tx2 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user1", "USD/2", big.NewInt(10)), + ).WithTimestamp(now).WithID(1) + + tx3 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user2", "USD/2", big.NewInt(10)), + ).WithTimestamp(now).WithID(2) + + require.NoError(t, insertTransactions(context.Background(), store, *tx1, *tx2, *tx3)) + + tx2FromDatabase, err := store.GetTransaction(context.Background(), tx2.ID) + require.NoError(t, err) + + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 0), + }, + "user1": { + "USD/2": core.NewVolumesInt64(0, 0), + }, + }, tx2FromDatabase.PreCommitVolumes) + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 10), + }, + "user1": { + "USD/2": core.NewVolumesInt64(10, 0), + }, + }, tx2FromDatabase.PostCommitVolumes) + + tx3FromDatabase, err := store.GetTransaction(context.Background(), tx3.ID) + require.NoError(t, err) + + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 10), + }, + "user2": { + "USD/2": core.NewVolumesInt64(0, 0), + }, + }, tx3FromDatabase.PreCommitVolumes) + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 20), + }, + "user2": { + "USD/2": core.NewVolumesInt64(10, 0), + }, + }, tx3FromDatabase.PostCommitVolumes) +} + +func TestInsertTwoTransactionAtSameDateInTwoBatch(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() + + tx1 := core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), + ).WithTimestamp(now.Add(-time.Hour)) + + tx2 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user1", "USD/2", big.NewInt(10)), + ).WithTimestamp(now).WithID(1) + + require.NoError(t, insertTransactions(context.Background(), store, *tx1, *tx2)) + + tx3 := core.NewTransaction().WithPostings( + core.NewPosting("bank", "user2", "USD/2", big.NewInt(10)), + ).WithTimestamp(now).WithID(2) + + require.NoError(t, insertTransactions(context.Background(), store, *tx3)) + + tx3FromDatabase, err := store.GetTransaction(context.Background(), tx3.ID) + require.NoError(t, err) + + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 10), + }, + "user2": { + "USD/2": core.NewVolumesInt64(0, 0), + }, + }, tx3FromDatabase.PreCommitVolumes) + RequireEqual(t, core.AccountsAssetsVolumes{ + "bank": { + "USD/2": core.NewVolumesInt64(100, 20), + }, + "user2": { + "USD/2": core.NewVolumesInt64(10, 0), + }, + }, tx3FromDatabase.PostCommitVolumes) } func TestListTransactions(t *testing.T) { + t.Parallel() store := newLedgerStore(t) + now := core.Now() - tx1 := core.ExpandTransactionFromEmptyPreCommitVolumes( - core.NewTransaction(). - WithID(0). - WithPostings( - core.NewPosting("world", "alice", "USD", big.NewInt(100)), - ). - WithTimestamp(now.Add(-3 * time.Hour)), - ) - tx2 := core.ExpandTransactionFromEmptyPreCommitVolumes( - core.NewTransaction(). - WithID(1). - WithPostings( - core.NewPosting("world", "bob", "USD", big.NewInt(100)), - ). - WithTimestamp(now.Add(-2 * time.Hour)), - ) - tx3 := core.ExpandTransactionFromEmptyPreCommitVolumes( - core.NewTransaction(). - WithID(2). - WithPostings( - core.NewPosting("world", "users:marley", "USD", big.NewInt(100)), - ). - WithTimestamp(now.Add(-time.Hour)), - ) - - require.NoError(t, store.InsertTransactions(context.Background(), tx1, tx2, tx3)) + tx1 := core.NewTransaction(). + WithID(0). + WithPostings( + core.NewPosting("world", "alice", "USD", big.NewInt(100)), + ). + WithTimestamp(now.Add(-3 * time.Hour)) + tx2 := core.NewTransaction(). + WithID(1). + WithPostings( + core.NewPosting("world", "bob", "USD", big.NewInt(100)), + ). + WithTimestamp(now.Add(-2 * time.Hour)) + tx3 := core.NewTransaction(). + WithID(2). + WithPostings( + core.NewPosting("world", "users:marley", "USD", big.NewInt(100)), + ). + WithTimestamp(now.Add(-time.Hour)) + + require.NoError(t, insertTransactions(context.Background(), store, *tx1, *tx2, *tx3)) type testCase struct { name string @@ -176,7 +720,7 @@ func TestListTransactions(t *testing.T) { expected: &api.Cursor[core.ExpandedTransaction]{ PageSize: 15, HasMore: false, - Data: []core.ExpandedTransaction{tx3, tx2, tx1}, + Data: Reverse(ExpandTransactions(tx1, tx2, tx3)...), }, }, { @@ -186,7 +730,7 @@ func TestListTransactions(t *testing.T) { expected: &api.Cursor[core.ExpandedTransaction]{ PageSize: 15, HasMore: false, - Data: []core.ExpandedTransaction{tx2}, + Data: ExpandTransactions(tx1, tx2)[1:], }, }, { @@ -196,7 +740,7 @@ func TestListTransactions(t *testing.T) { expected: &api.Cursor[core.ExpandedTransaction]{ PageSize: 15, HasMore: false, - Data: []core.ExpandedTransaction{tx3}, + Data: ExpandTransactions(tx1, tx2, tx3)[2:], }, }, } @@ -206,7 +750,7 @@ func TestListTransactions(t *testing.T) { t.Run(tc.name, func(t *testing.T) { cursor, err := store.GetTransactions(context.Background(), tc.query) require.NoError(t, err) - require.Equal(t, *tc.expected, *cursor) + RequireEqual(t, *tc.expected, *cursor) count, err := store.CountTransactions(context.Background(), tc.query) require.NoError(t, err) @@ -214,3 +758,47 @@ func TestListTransactions(t *testing.T) { }) } } + +func TestInsertTransactionsWithConflict(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + + now := core.Now() + + tx1 := core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), + ).WithTimestamp(now) + tx2 := core.NewTransaction().WithPostings( + core.NewPosting("world", "bank", "USD/2", big.NewInt(100)), + ).WithID(1).WithTimestamp(now.Add(time.Minute)) + + require.NoError(t, insertTransactions(context.Background(), store, *tx1, *tx2)) + + checkTx2 := func() { + tx2FromDB, err := store.GetTransaction(context.Background(), tx2.ID) + require.NoError(t, err) + require.Equal(t, core.ExpandedTransaction{ + Transaction: *tx2, + PreCommitVolumes: core.AccountsAssetsVolumes{ + "world": map[string]*core.Volumes{ + "USD/2": core.NewVolumesInt64(0, 100), + }, + "bank": map[string]*core.Volumes{ + "USD/2": core.NewVolumesInt64(100, 0), + }, + }, + PostCommitVolumes: core.AccountsAssetsVolumes{ + "world": map[string]*core.Volumes{ + "USD/2": core.NewVolumesInt64(0, 200), + }, + "bank": map[string]*core.Volumes{ + "USD/2": core.NewVolumesInt64(200, 0), + }, + }, + }, *tx2FromDB) + } + + checkTx2() + require.NoError(t, insertTransactions(context.Background(), store, *tx1)) + checkTx2() +} diff --git a/components/ledger/pkg/storage/ledgerstore/utils.go b/components/ledger/pkg/storage/ledgerstore/utils.go deleted file mode 100644 index 6c3f9ddc64..0000000000 --- a/components/ledger/pkg/storage/ledgerstore/utils.go +++ /dev/null @@ -1,5 +0,0 @@ -package ledgerstore - -func ptr[T any](t T) *T { - return &t -} diff --git a/components/ledger/pkg/storage/ledgerstore/volumes.go b/components/ledger/pkg/storage/ledgerstore/volumes.go index fb62245aa2..9be4e08e49 100644 --- a/components/ledger/pkg/storage/ledgerstore/volumes.go +++ b/components/ledger/pkg/storage/ledgerstore/volumes.go @@ -2,114 +2,33 @@ package ledgerstore import ( "context" + "fmt" "math/big" - "strings" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" - "github.com/uptrace/bun" + storageerrors "github.com/formancehq/ledger/pkg/storage" ) -const ( - volumesTableName = "volumes" -) - -type Volumes struct { - bun.BaseModel `bun:"volumes,alias:volumes"` - - Account string `bun:"account,type:varchar,unique:account_asset"` - AccountJson []string `bun:"account_json,type:jsonb"` - Asset string `bun:"asset,type:varchar,unique:account_asset"` - Input uint64 `bun:"input,type:numeric"` - Output uint64 `bun:"output,type:numeric"` -} - -func (s *Store) UpdateVolumes(ctx context.Context, volumes ...core.AccountsAssetsVolumes) error { - if !s.isInitialized { - return storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "update_volumes") - defer recordMetrics() - - volumesMap := make(map[string]*Volumes) - for _, vs := range volumes { - for account, accountVolumes := range vs { - for asset, volumes := range accountVolumes { - - // De-duplicate same volumes to only have the last version - volumesMap[account+asset] = &Volumes{ - Account: account, - AccountJson: strings.Split(account, ":"), - Asset: asset, - Input: volumes.Input.Uint64(), - Output: volumes.Output.Uint64(), - } - } - } - } - - vls := make([]*Volumes, 0, len(volumes)) - for _, v := range volumesMap { - vls = append(vls, v) - } +func (s *Store) GetAssetsVolumes(ctx context.Context, accountAddress string) (core.VolumesByAssets, error) { + moves := make([]Move, 0) - query := s.schema.NewInsert(volumesTableName). - Model(&vls). - On("CONFLICT (account, asset) DO UPDATE"). - Set("input = EXCLUDED.input, output = EXCLUDED.output"). - String() - - _, err := s.schema.ExecContext(ctx, query) - return storageerrors.PostgresError(err) -} - -func (s *Store) GetAssetsVolumes(ctx context.Context, accountAddress string) (core.AssetsVolumes, error) { - if !s.isInitialized { - return nil, storageerrors.StorageError(storageerrors.ErrStoreNotInitialized) - } - recordMetrics := s.instrumentalized(ctx, "get_assets_volumes") - defer recordMetrics() - - query := s.schema.NewSelect(volumesTableName). - Model((*Volumes)(nil)). - Column("asset", "input", "output"). + err := s.schema.NewSelect(MovesTableName). + Model(&moves). Where("account = ?", accountAddress). - String() - - rows, err := s.schema.QueryContext(ctx, query) + ColumnExpr("asset"). + ColumnExpr(fmt.Sprintf(`"%s".first(post_commit_input_value order by timestamp desc) as post_commit_input_value`, s.schema.Name())). + ColumnExpr(fmt.Sprintf(`"%s".first(post_commit_output_value order by timestamp desc) as post_commit_output_value`, s.schema.Name())). + GroupExpr("account, asset"). + Scan(ctx) if err != nil { return nil, storageerrors.PostgresError(err) } - defer rows.Close() - volumes := core.AssetsVolumes{} - for rows.Next() { - var ( - asset string - inputStr string - outputStr string - ) - if err := rows.Scan(&asset, &inputStr, &outputStr); err != nil { - return nil, storageerrors.PostgresError(err) - } - - input, ok := new(big.Int).SetString(inputStr, 10) - if !ok { - panic("unable to restore big int") - } - - output, ok := new(big.Int).SetString(outputStr, 10) - if !ok { - panic("unable to restore big int") - } - - volumes[asset] = core.Volumes{ - Input: input, - Output: output, - } - } - if err := rows.Err(); err != nil { - return nil, storageerrors.PostgresError(err) + volumes := core.VolumesByAssets{} + for _, move := range moves { + volumes[move.Asset] = core.NewEmptyVolumes(). + WithInput((*big.Int)(move.PostCommitInputVolume)). + WithOutput((*big.Int)(move.PostCommitOutputVolume)) } return volumes, nil diff --git a/components/ledger/pkg/storage/ledgerstore/volumes_test.go b/components/ledger/pkg/storage/ledgerstore/volumes_test.go index 56f31f7a21..64afabae98 100644 --- a/components/ledger/pkg/storage/ledgerstore/volumes_test.go +++ b/components/ledger/pkg/storage/ledgerstore/volumes_test.go @@ -4,87 +4,47 @@ import ( "context" "math/big" "testing" + "time" "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/storage" - "github.com/formancehq/ledger/pkg/storage/sqlstoragetesting" "github.com/stretchr/testify/require" ) -func TestVolumes(t *testing.T) { - d := sqlstoragetesting.StorageDriver(t) - - defer func(d *storage.Driver, ctx context.Context) { - require.NoError(t, d.Close(ctx)) - }(d, context.Background()) - - store, err := d.CreateLedgerStore(context.Background(), "foo") - require.NoError(t, err) - - _, err = store.Migrate(context.Background()) - require.NoError(t, err) - - t.Run("success update volumes", func(t *testing.T) { - foo := core.AssetsVolumes{ - "bar": { - Input: big.NewInt(1), - Output: big.NewInt(2), - }, - } - - foo2 := core.AssetsVolumes{ - "bar2": { - Input: big.NewInt(3), - Output: big.NewInt(4), - }, - } - - volumes := core.AccountsAssetsVolumes{ - "foo": foo, - "foo2": foo2, - } - - err := store.UpdateVolumes(context.Background(), volumes) - require.NoError(t, err, "update volumes should not fail") - - assetVolumes, err := store.GetAssetsVolumes(context.Background(), "foo") - require.NoError(t, err, "get asset volumes should not fail") - require.Equal(t, foo, assetVolumes, "asset volumes should be equal") - - assetVolumes, err = store.GetAssetsVolumes(context.Background(), "foo2") - require.NoError(t, err, "get asset volumes should not fail") - require.Equal(t, foo2, assetVolumes, "asset volumes should be equal") - }) - - t.Run("success update same volume", func(t *testing.T) { - foo := core.AssetsVolumes{ - "bar": { - Input: big.NewInt(1), - Output: big.NewInt(2), - }, - } - - foo2 := core.AssetsVolumes{ - "bar": { - Input: big.NewInt(3), - Output: big.NewInt(4), - }, - } - - volumes := []core.AccountsAssetsVolumes{ - { - "foo": foo, - }, - { - "foo": foo2, - }, - } - - err := store.UpdateVolumes(context.Background(), volumes...) - require.NoError(t, err, "update volumes should not fail") - - assetVolumes, err := store.GetAssetsVolumes(context.Background(), "foo") - require.NoError(t, err, "get asset volumes should not fail") - require.Equal(t, foo2, assetVolumes, "asset volumes should be equal") - }) +func TestGetAssetsVolumes(t *testing.T) { + t.Parallel() + store := newLedgerStore(t) + now := core.Now() + + tx1 := core.NewTransaction(). + WithID(0). + WithPostings( + core.NewPosting("world", "alice", "USD", big.NewInt(100)), + ). + WithTimestamp(now.Add(-3 * time.Hour)) + tx2 := core.NewTransaction(). + WithID(1). + WithPostings( + core.NewPosting("world", "bob", "USD", big.NewInt(100)), + ). + WithTimestamp(now.Add(-2 * time.Hour)) + tx3 := core.NewTransaction(). + WithID(2). + WithPostings( + core.NewPosting("world", "users:marley", "USD", big.NewInt(100)), + ). + WithTimestamp(now.Add(-time.Hour)) + + require.NoError(t, insertTransactions(context.Background(), store, *tx1, *tx2, *tx3)) + + assetVolumesForWorld, err := store.GetAssetsVolumes(context.Background(), "world") + require.NoError(t, err, "get asset volumes should not fail") + require.Equal(t, core.VolumesByAssets{ + "USD": core.NewEmptyVolumes().WithOutputInt64(300), + }, assetVolumesForWorld, "asset volumes should be equal") + + assetVolumesForBob, err := store.GetAssetsVolumes(context.Background(), "bob") + require.NoError(t, err, "get asset volumes should not fail") + require.Equal(t, core.VolumesByAssets{ + "USD": core.NewEmptyVolumes().WithInputInt64(100), + }, assetVolumesForBob, "asset volumes should be equal") } diff --git a/components/ledger/pkg/storage/migrations/migrations.go b/components/ledger/pkg/storage/migrations/migrations.go index 50e0e32270..0ca83fa894 100644 --- a/components/ledger/pkg/storage/migrations/migrations.go +++ b/components/ledger/pkg/storage/migrations/migrations.go @@ -15,8 +15,7 @@ import ( "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/opentelemetry/tracer" - sqlerrors "github.com/formancehq/ledger/pkg/storage/errors" - "github.com/formancehq/ledger/pkg/storage/schema" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/pkg/errors" "github.com/uptrace/bun" @@ -24,42 +23,42 @@ import ( const migrationsTableName = "migrations_v2" -type MigrationsTable struct { +type table struct { bun.BaseModel `bun:"migrations_v2,alias:migrations_v2"` Version string `bun:"version,type:varchar,unique"` Date string `bun:"date,type:varchar"` } -func createMigrationsTable(ctx context.Context, schema schema.Schema) error { +func createMigrationsTable(ctx context.Context, schema storage.Schema) error { _, err := schema.NewCreateTable(migrationsTableName). - Model((*MigrationsTable)(nil)). + Model((*table)(nil)). IfNotExists(). Exec(ctx) return err } -func Migrate(ctx context.Context, s schema.Schema, migrations ...Migration) (bool, error) { +func Migrate(ctx context.Context, s storage.Schema, migrations ...Migration) (bool, error) { ctx, span := tracer.Start(ctx, "Migrate") defer span.End() if err := createMigrationsTable(ctx, s); err != nil { - return false, sqlerrors.PostgresError(err) + return false, storage.PostgresError(err) } tx, err := s.BeginTx(ctx, &sql.TxOptions{}) if err != nil { - return false, sqlerrors.PostgresError(err) + return false, storage.PostgresError(err) } - defer func(tx *schema.Tx) { + defer func(tx *storage.Tx) { _ = tx.Rollback() }(tx) modified := false for _, m := range migrations { sb := s.NewSelect(migrationsTableName). - Model((*MigrationsTable)(nil)). + Model((*table)(nil)). Column("version"). Where("version = ?", m.Version) @@ -77,7 +76,7 @@ func Migrate(ctx context.Context, s schema.Schema, migrations ...Migration) (boo logging.FromContext(ctx).Debugf("running migration %s", m.Version) - handlersForCurrentEngine, ok := m.Handlers[s.Flavor()] + handlersForCurrentEngine, ok := m.Handlers["postgres"] if ok { for _, h := range handlersForCurrentEngine { err := h(ctx, s, tx) @@ -89,15 +88,15 @@ func Migrate(ctx context.Context, s schema.Schema, migrations ...Migration) (boo handlersForAnyEngine, ok := m.Handlers["any"] if ok { - for _, h := range handlersForAnyEngine { - err := h(ctx, s, tx) + for num, h := range handlersForAnyEngine { + err := h(logging.ContextWithField(ctx, "migrations", num), s, tx) if err != nil { return false, err } } } - m := MigrationsTable{ + m := table{ Version: m.Version, Date: core.Now().Format(time.RFC3339), } @@ -105,17 +104,17 @@ func Migrate(ctx context.Context, s schema.Schema, migrations ...Migration) (boo if _, err := tx.ExecContext(ctx, sbInsert.String()); err != nil { logging.FromContext(ctx).Errorf("failed to insert migration version %s: %s", m.Version, err) - return false, sqlerrors.PostgresError(err) + return false, storage.PostgresError(err) } } - return modified, sqlerrors.PostgresError(tx.Commit()) + return modified, storage.PostgresError(tx.Commit()) } -func GetMigrations(ctx context.Context, schema schema.Schema) ([]core.MigrationInfo, error) { +func GetMigrations(ctx context.Context, schema storage.Schema) ([]core.MigrationInfo, error) { sb := schema.NewSelect(migrationsTableName). - Model((*MigrationsTable)(nil)). + Model((*table)(nil)). Column("version", "date") rows, err := schema.QueryContext(ctx, sb.String()) @@ -244,7 +243,7 @@ func CollectMigrationFiles(migrationsFS fs.FS) ([]Migration, error) { } func SQLMigrationFunc(content string) MigrationFunc { - return func(ctx context.Context, schema schema.Schema, tx *schema.Tx) error { + return func(ctx context.Context, schema storage.Schema, tx *storage.Tx) error { plain := strings.ReplaceAll(content, "VAR_LEDGER_NAME", schema.Name()) r := regexp.MustCompile(`[\n\t\s]+`) plain = r.ReplaceAllString(plain, " ") @@ -256,7 +255,7 @@ func SQLMigrationFunc(content string) MigrationFunc { var RegisteredGoMigrations []Migration -type MigrationFunc func(ctx context.Context, schema schema.Schema, tx *schema.Tx) error +type MigrationFunc func(ctx context.Context, schema storage.Schema, tx *storage.Tx) error type HandlersByEngine map[string][]MigrationFunc diff --git a/components/ledger/pkg/storage/migrations/migrations_test.go b/components/ledger/pkg/storage/migrations/migrations_test.go index 0e91f4c017..e31c8e9e01 100644 --- a/components/ledger/pkg/storage/migrations/migrations_test.go +++ b/components/ledger/pkg/storage/migrations/migrations_test.go @@ -8,10 +8,9 @@ import ( "testing" "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/ledger/pkg/storage/migrations" - "github.com/formancehq/ledger/pkg/storage/schema" - "github.com/formancehq/ledger/pkg/storage/utils" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/psanford/memfs" @@ -46,7 +45,7 @@ func TestCollectMigrations(t *testing.T) { NO SQL; `), 0666)) - migrations.RegisterGoMigrationFromFilename("migrates/0-first-migration/sqlite.go", func(ctx context.Context, schema schema.Schema, tx *schema.Tx) error { + migrations.RegisterGoMigrationFromFilename("migrates/0-first-migration/sqlite.go", func(ctx context.Context, schema storage.Schema, tx *storage.Tx) error { return nil }) @@ -86,14 +85,15 @@ func TestMigrationsOrders(t *testing.T) { func TestMigrates(t *testing.T) { pgServer := pgtesting.NewPostgresDatabase(t) - sqlDB, err := utils.OpenSQLDB(utils.ConnectionOptions{ + sqlDB, err := storage.OpenSQLDB(storage.ConnectionOptions{ DatabaseSourceName: pgServer.ConnString(), Debug: testing.Verbose(), + Trace: testing.Verbose(), }) if err != nil { t.Fatal(err) } - db := schema.NewPostgresDB(sqlDB) + db := storage.NewDatabase(sqlDB) s, err := db.Schema("testing") require.NoError(t, err) @@ -140,7 +140,7 @@ func TestMigrates(t *testing.T) { }, Handlers: migrations.HandlersByEngine{ "any": { - func(ctx context.Context, schema schema.Schema, tx *schema.Tx) error { + func(ctx context.Context, schema storage.Schema, tx *storage.Tx) error { sb := s.NewUpdate(ledgerstore.TransactionsTableName). Model((*ledgerstore.Transaction)(nil)). Set("timestamp = ?", core.Now()). @@ -162,7 +162,7 @@ func TestMigrates(t *testing.T) { } func TestRegister(t *testing.T) { - fn := func(ctx context.Context, schema schema.Schema, tx *schema.Tx) error { + fn := func(ctx context.Context, schema storage.Schema, tx *storage.Tx) error { return nil } diff --git a/components/ledger/pkg/storage/opentelemetry/metrics/metrics.go b/components/ledger/pkg/storage/opentelemetry/metrics/metrics.go deleted file mode 100644 index d421730e36..0000000000 --- a/components/ledger/pkg/storage/opentelemetry/metrics/metrics.go +++ /dev/null @@ -1,27 +0,0 @@ -package metrics - -import ( - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" -) - -type SQLStorageMetricsRegistry struct { - Latencies instrument.Int64Histogram -} - -func RegisterSQLStorageMetrics(schemaName string) (*SQLStorageMetricsRegistry, error) { - meter := global.MeterProvider().Meter(schemaName) - - latencies, err := meter.Int64Histogram( - "ledger.storage.sql.time", - instrument.WithUnit("ms"), - instrument.WithDescription("Latency of SQL calls"), - ) - if err != nil { - return nil, err - } - - return &SQLStorageMetricsRegistry{ - Latencies: latencies, - }, nil -} diff --git a/components/ledger/pkg/storage/schema/schema.go b/components/ledger/pkg/storage/schema.go similarity index 55% rename from components/ledger/pkg/storage/schema/schema.go rename to components/ledger/pkg/storage/schema.go index ba6b2dc3d3..558f909413 100644 --- a/components/ledger/pkg/storage/schema/schema.go +++ b/components/ledger/pkg/storage/schema.go @@ -1,14 +1,18 @@ -package schema +package storage import ( "context" "database/sql" "fmt" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" "github.com/uptrace/bun" ) +const ( + createSchemaQuery = `CREATE SCHEMA IF NOT EXISTS "%s"` + deleteSchemaQuery = `DROP SCHEMA "%s" CASCADE` +) + type Schema struct { bun.IDB name string @@ -29,28 +33,20 @@ func (s *Schema) Table(name string) string { return fmt.Sprintf(`"%s".%s`, s.name, name) } -const ( - createSchemaQuery = `CREATE SCHEMA IF NOT EXISTS "%s"` -) - func (s *Schema) Create(ctx context.Context) error { _, err := s.ExecContext(ctx, fmt.Sprintf(createSchemaQuery, s.name)) - return storageerrors.PostgresError(err) + return PostgresError(err) } -const ( - deleteSchemaQuery = `DROP SCHEMA "%s" CASCADE` -) - func (s *Schema) Delete(ctx context.Context) error { _, err := s.ExecContext(ctx, fmt.Sprintf(deleteSchemaQuery, s.name)) - return storageerrors.PostgresError(err) + return PostgresError(err) } func (s *Schema) BeginTx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) { bunTx, err := s.IDB.BeginTx(ctx, opts) if err != nil { - return nil, storageerrors.PostgresError(err) + return nil, PostgresError(err) } return &Tx{ schema: s, @@ -58,10 +54,6 @@ func (s *Schema) BeginTx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) }, nil } -func (s *Schema) Flavor() string { - return "postgres" -} - // Override all bun methods to use the schema name func (s *Schema) NewInsert(tableName string) *bun.InsertQuery { @@ -83,55 +75,3 @@ func (s *Schema) NewCreateTable(tableName string) *bun.CreateTableQuery { func (s *Schema) NewDelete(tableName string) *bun.DeleteQuery { return s.IDB.NewDelete().ModelTableExpr("?0.?1", bun.Ident(s.Name()), bun.Ident(tableName)) } - -type DB interface { - Initialize(ctx context.Context) error - Schema(name string) (Schema, error) - Close(ctx context.Context) error -} - -type postgresDB struct { - db *bun.DB -} - -func (p *postgresDB) Initialize(ctx context.Context) error { - _, err := p.db.ExecContext(ctx, "CREATE EXTENSION IF NOT EXISTS pgcrypto") - if err != nil { - return storageerrors.PostgresError(err) - } - _, err = p.db.ExecContext(ctx, "CREATE EXTENSION IF NOT EXISTS pg_trgm") - if err != nil { - return storageerrors.PostgresError(err) - } - return nil -} - -func (p *postgresDB) Schema(name string) (Schema, error) { - return Schema{ - IDB: p.db, - name: name, - }, nil -} - -func (p *postgresDB) Close(ctx context.Context) error { - return p.db.Close() -} - -func NewPostgresDB(db *bun.DB) *postgresDB { - return &postgresDB{ - db: db, - } -} - -type Tx struct { - schema *Schema - bun.Tx -} - -func (s *Tx) NewSelect(tableName string) *bun.SelectQuery { - return s.Tx.NewSelect().ModelTableExpr("?0.?1 as ?1", bun.Ident(s.schema.Name()), bun.Ident(tableName)) -} - -func (s *Tx) NewInsert(tableName string) *bun.InsertQuery { - return s.Tx.NewInsert().ModelTableExpr("?0.?1 as ?1", bun.Ident(s.schema.Name()), bun.Ident(tableName)) -} diff --git a/components/ledger/pkg/storage/sqlstoragetesting/storage.go b/components/ledger/pkg/storage/storagetesting/storage.go similarity index 64% rename from components/ledger/pkg/storage/sqlstoragetesting/storage.go rename to components/ledger/pkg/storage/storagetesting/storage.go index 19d936ced9..77c67fc7b0 100644 --- a/components/ledger/pkg/storage/sqlstoragetesting/storage.go +++ b/components/ledger/pkg/storage/storagetesting/storage.go @@ -1,4 +1,4 @@ -package sqlstoragetesting +package storagetesting import ( "context" @@ -6,17 +6,16 @@ import ( "time" "github.com/formancehq/ledger/pkg/storage" + "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/ledger/pkg/storage/ledgerstore" - "github.com/formancehq/ledger/pkg/storage/schema" - "github.com/formancehq/ledger/pkg/storage/utils" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" ) -func StorageDriver(t pgtesting.TestingT) *storage.Driver { +func StorageDriver(t pgtesting.TestingT) *driver.Driver { pgServer := pgtesting.NewPostgresDatabase(t) - db, err := utils.OpenSQLDB(utils.ConnectionOptions{ + db, err := storage.OpenSQLDB(storage.ConnectionOptions{ DatabaseSourceName: pgServer.ConnString(), Debug: testing.Verbose(), MaxIdleConns: 40, @@ -29,7 +28,7 @@ func StorageDriver(t pgtesting.TestingT) *storage.Driver { db.Close() }) - d := storage.NewDriver("postgres", schema.NewPostgresDB(db), ledgerstore.DefaultStoreConfig) + d := driver.New("postgres", storage.NewDatabase(db), ledgerstore.DefaultStoreConfig) require.NoError(t, d.Initialize(context.Background())) diff --git a/components/ledger/pkg/storage/systemstore/configuration.go b/components/ledger/pkg/storage/systemstore/configuration.go index 3196f3180b..4a9967e2f4 100644 --- a/components/ledger/pkg/storage/systemstore/configuration.go +++ b/components/ledger/pkg/storage/systemstore/configuration.go @@ -4,7 +4,7 @@ import ( "context" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/uptrace/bun" ) diff --git a/components/ledger/pkg/storage/systemstore/ledgers.go b/components/ledger/pkg/storage/systemstore/ledgers.go index 1b3770b97a..cb3c64fa10 100644 --- a/components/ledger/pkg/storage/systemstore/ledgers.go +++ b/components/ledger/pkg/storage/systemstore/ledgers.go @@ -4,7 +4,7 @@ import ( "context" "github.com/formancehq/ledger/pkg/core" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" + storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/pkg/errors" "github.com/uptrace/bun" ) diff --git a/components/ledger/pkg/storage/systemstore/store.go b/components/ledger/pkg/storage/systemstore/store.go index 96468424c3..0a3097eea9 100644 --- a/components/ledger/pkg/storage/systemstore/store.go +++ b/components/ledger/pkg/storage/systemstore/store.go @@ -3,22 +3,21 @@ package systemstore import ( "context" - storageerrors "github.com/formancehq/ledger/pkg/storage/errors" - "github.com/formancehq/ledger/pkg/storage/schema" + "github.com/formancehq/ledger/pkg/storage" ) type Store struct { - schema schema.Schema + schema storage.Schema } -func NewStore(schema schema.Schema) *Store { +func NewStore(schema storage.Schema) *Store { return &Store{schema: schema} } func (s *Store) Initialize(ctx context.Context) error { if err := s.CreateLedgersTable(ctx); err != nil { - return storageerrors.PostgresError(err) + return storage.PostgresError(err) } - return storageerrors.PostgresError(s.CreateConfigurationTable(ctx)) + return storage.PostgresError(s.CreateConfigurationTable(ctx)) } diff --git a/components/ledger/pkg/storage/tx.go b/components/ledger/pkg/storage/tx.go new file mode 100644 index 0000000000..f925c508dc --- /dev/null +++ b/components/ledger/pkg/storage/tx.go @@ -0,0 +1,23 @@ +package storage + +import ( + "github.com/uptrace/bun" +) + +type Tx struct { + schema *Schema + bun.Tx +} + +func (s *Tx) NewSelect(tableName string) *bun.SelectQuery { + return s.Tx.NewSelect().ModelTableExpr("?0.?1 as ?1", bun.Ident(s.schema.Name()), bun.Ident(tableName)) +} + +func (s *Tx) NewInsert(tableName string) *bun.InsertQuery { + return s.Tx.NewInsert().ModelTableExpr("?0.?1 as ?1", bun.Ident(s.schema.Name()), bun.Ident(tableName)) +} + +func (s *Tx) NewUpdate(tableName string) *bun.UpdateQuery { + return s.Tx.NewUpdate().ModelTableExpr("?0.?1 as ?1", bun.Ident(s.schema.Name()), bun.Ident(tableName)) +} + diff --git a/components/ledger/pkg/storage/utils/utils.go b/components/ledger/pkg/storage/utils.go similarity index 78% rename from components/ledger/pkg/storage/utils/utils.go rename to components/ledger/pkg/storage/utils.go index f3d93e3997..2d05535991 100644 --- a/components/ledger/pkg/storage/utils/utils.go +++ b/components/ledger/pkg/storage/utils.go @@ -1,4 +1,4 @@ -package utils +package storage import ( "database/sql" @@ -9,18 +9,20 @@ import ( "github.com/uptrace/bun" "github.com/uptrace/bun/dialect/pgdialect" "github.com/uptrace/bun/extra/bundebug" + "github.com/uptrace/bun/extra/bunotel" ) type ConnectionOptions struct { DatabaseSourceName string Debug bool + Trace bool Writer io.Writer MaxIdleConns int MaxOpenConns int ConnMaxIdleTime time.Duration } -func OpenSQLDB(options ConnectionOptions) (*bun.DB, error) { +func OpenSQLDB(options ConnectionOptions, hooks ...bun.QueryHook) (*bun.DB, error) { sqldb, err := sql.Open("postgres", options.DatabaseSourceName) if err != nil { return nil, err @@ -36,7 +38,7 @@ func OpenSQLDB(options ConnectionOptions) (*bun.DB, error) { } db := bun.NewDB(sqldb, pgdialect.New()) - if options.Debug { + if options.Trace { writer := options.Writer if writer == nil { writer = os.Stdout @@ -46,6 +48,10 @@ func OpenSQLDB(options ConnectionOptions) (*bun.DB, error) { bundebug.WithWriter(writer), )) } + db.AddQueryHook(bunotel.NewQueryHook()) + for _, hook := range hooks { + db.AddQueryHook(hook) + } if err := db.Ping(); err != nil { return nil, err diff --git a/components/search/benthos/streams/ledger_ingestion.yaml b/components/search/benthos/streams/ledger_ingestion.yaml index 055e5abd68..236a778861 100644 --- a/components/search/benthos/streams/ledger_ingestion.yaml +++ b/components/search/benthos/streams/ledger_ingestion.yaml @@ -27,26 +27,6 @@ pipeline: }).values()).values().flatten() } - map volumes { - root = this.map_each(v -> v.value.map_each(v2 -> { - "action": "index", - "id": "%s-%s".format(v.key, v2.key), - "document": { - "data": { - "name": v2.key, - "input": v2.value.input, - "output": v2.value.output, - "account": v.key - }, - "indexed": { - "account": v.key, - "name": v2.key - }, - "kind": "ASSET" - } - }).values()).values().flatten() - } - map tx { root = { "action": "index", @@ -83,16 +63,31 @@ pipeline: this.payload.transactions.map_each(t -> t.apply("tx")).map_each(t -> t.assign({ "id": "TRANSACTION-%s-%s".format(this.payload.ledger, t.id) })), - this.payload.volumes.apply("volumes"). - sort(v -> v.right.id > v.left.id). - map_each(t -> t.assign({ - "id": "ASSET-%s-%s".format(this.payload.ledger, t.id) - })), - this.payload.volumes.apply("account"). - sort(v -> v.right.id > v.left.id). - map_each(t -> t.assign({ - "id": "ACCOUNT-%s-%s".format(this.payload.ledger, t.id) - })), + this.payload.transactions.map_each(t -> t.postings.map_each(p -> [{ + "action": "upsert", + "id": "ACCOUNT-%s-%s".format(this.payload.ledger, p.source), + "document": { + "data": { + "address": p.source + }, + "indexed": { + "address": p.source + }, + "kind": "ACCOUNT" + } + }, { + "action": "upsert", + "id": "ACCOUNT-%s-%s".format(this.payload.ledger, p.destination), + "document": { + "data": { + "address": p.destination + }, + "indexed": { + "address": p.destination + }, + "kind": "ACCOUNT" + } + }])).flatten().flatten() ].flatten().map_each(t -> t.merge({ "document": { "when": this.date, diff --git a/libs/events/v1/ledger/COMMITTED_TRANSACTIONS.yaml b/libs/events/v1/ledger/COMMITTED_TRANSACTIONS.yaml index caf51d8a64..63d623c48d 100644 --- a/libs/events/v1/ledger/COMMITTED_TRANSACTIONS.yaml +++ b/libs/events/v1/ledger/COMMITTED_TRANSACTIONS.yaml @@ -33,32 +33,6 @@ properties: required: [] txid: type: number - preCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number - postCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number timestamp: type: string required: @@ -66,49 +40,7 @@ properties: - reference - metadata - txid - - preCommitVolumes - - postCommitVolumes - timestamp - volumes: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number - postCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number - preCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number required: - ledger - transactions -- volumes -- postCommitVolumes -- preCommitVolumes diff --git a/libs/events/v1/ledger/REVERTED_TRANSACTION.yaml b/libs/events/v1/ledger/REVERTED_TRANSACTION.yaml index f9c39f5f08..9f7fbe9b42 100644 --- a/libs/events/v1/ledger/REVERTED_TRANSACTION.yaml +++ b/libs/events/v1/ledger/REVERTED_TRANSACTION.yaml @@ -31,32 +31,6 @@ properties: required: [ ] txid: type: number - preCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number - postCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number timestamp: type: string required: @@ -64,8 +38,6 @@ properties: - reference - metadata - txid - - preCommitVolumes - - postCommitVolumes - timestamp revertTransaction: type: object @@ -96,32 +68,6 @@ properties: required: [ ] txid: type: number - preCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number - postCommitVolumes: - type: object - additionalProperties: - type: object - additionalProperties: - type: object - properties: - input: - type: number - output: - type: number - balance: - type: number timestamp: type: string required: @@ -129,8 +75,6 @@ properties: - reference - metadata - txid - - preCommitVolumes - - postCommitVolumes - timestamp required: - ledger diff --git a/libs/go-libs/api/response_utils.go b/libs/go-libs/api/response_utils.go new file mode 100644 index 0000000000..f861f27caf --- /dev/null +++ b/libs/go-libs/api/response_utils.go @@ -0,0 +1,37 @@ +package api + +import ( + "bytes" + "encoding/json" + "io" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Encode(t require.TestingT, v interface{}) []byte { + data, err := json.Marshal(v) + assert.NoError(t, err) + return data +} + +func Buffer(t require.TestingT, v interface{}) *bytes.Buffer { + return bytes.NewBuffer(Encode(t, v)) +} + +func Decode(t require.TestingT, reader io.Reader, v interface{}) { + err := json.NewDecoder(reader).Decode(v) + require.NoError(t, err) +} + +func DecodeSingleResponse[T any](t require.TestingT, reader io.Reader) (T, bool) { + res := BaseResponse[T]{} + Decode(t, reader, &res) + return *res.Data, true +} + +func DecodeCursorResponse[T any](t require.TestingT, reader io.Reader) *Cursor[T] { + res := BaseResponse[T]{} + Decode(t, reader, &res) + return res.Cursor +} diff --git a/libs/go-libs/collectionutils/linked_list.go b/libs/go-libs/collectionutils/linked_list.go new file mode 100644 index 0000000000..b6ecb8ff97 --- /dev/null +++ b/libs/go-libs/collectionutils/linked_list.go @@ -0,0 +1,142 @@ +package collectionutils + +import ( + "sync" +) + +type LinkedListNode[T any] struct { + object T + list *LinkedList[T] + previousNode, nextNode *LinkedListNode[T] +} + +func (n *LinkedListNode[T]) Next() *LinkedListNode[T] { + return n.nextNode +} + +func (n *LinkedListNode[T]) Value() T { + return n.object +} + +func (n *LinkedListNode[T]) Remove() { + if n.previousNode != nil { + n.previousNode.nextNode = n.nextNode + } + if n.nextNode != nil { + n.nextNode.previousNode = n.previousNode + } + if n == n.list.firstNode { + n.list.firstNode = n.nextNode + } + if n == n.list.lastNode { + n.list.lastNode = n.previousNode + } +} + +type LinkedList[T any] struct { + mu sync.Mutex + firstNode, lastNode *LinkedListNode[T] +} + +func (r *LinkedList[T]) Append(objects ...T) { + r.mu.Lock() + defer r.mu.Unlock() + + for _, object := range objects { + if r.firstNode == nil { + r.firstNode = &LinkedListNode[T]{ + object: object, + list: r, + } + r.lastNode = r.firstNode + continue + } + r.lastNode = &LinkedListNode[T]{ + object: object, + previousNode: r.lastNode, + list: r, + } + r.lastNode.previousNode.nextNode = r.lastNode + } +} + +func (r *LinkedList[T]) RemoveFirst(cmp func(T) bool) *LinkedListNode[T] { + r.mu.Lock() + defer r.mu.Unlock() + + node := r.firstNode + for node != nil { + if cmp(node.object) { + node.Remove() + return node + } + node = node.nextNode + } + + return nil +} + +func (r *LinkedList[T]) RemoveValue(t T) *LinkedListNode[T] { + return r.RemoveFirst(func(t2 T) bool { + return (any)(t) == (any)(t2) + }) +} + +func (r *LinkedList[T]) TakeFirst() T { + var t T + if r.firstNode == nil { + return t + } + ret := r.firstNode.object + if r.firstNode.nextNode == nil { + r.firstNode = nil + } else { + r.firstNode = r.firstNode.nextNode + r.firstNode.previousNode = nil + } + return ret +} + +func (r *LinkedList[T]) Length() int { + r.mu.Lock() + defer r.mu.Unlock() + + count := 0 + + node := r.firstNode + for node != nil { + count++ + node = node.nextNode + } + + return count +} + +func (r *LinkedList[T]) ForEach(f func(t T)) { + r.mu.Lock() + defer r.mu.Unlock() + + node := r.firstNode + for node != nil { + f(node.object) + node = node.nextNode + } +} + +func (r *LinkedList[T]) Slice() []T { + ret := make([]T, 0) + node := r.firstNode + for node != nil { + ret = append(ret, node.object) + node = node.nextNode + } + return ret +} + +func (r *LinkedList[T]) FirstNode() *LinkedListNode[T] { + return r.firstNode +} + +func NewLinkedList[T any]() *LinkedList[T] { + return &LinkedList[T]{} +} diff --git a/libs/go-libs/collectionutils/slice.go b/libs/go-libs/collectionutils/slice.go index 83cb1e8c44..be5f4a55d8 100644 --- a/libs/go-libs/collectionutils/slice.go +++ b/libs/go-libs/collectionutils/slice.go @@ -22,6 +22,14 @@ func Filter[TYPE any](input []TYPE, filter func(TYPE) bool) []TYPE { return ret } +func Flatten[TYPE any](input [][]TYPE) []TYPE { + ret := make([]TYPE, 0) + for _, types := range input { + ret = append(ret, types...) + } + return ret +} + func First[TYPE any](input []TYPE, filter func(TYPE) bool) TYPE { var zero TYPE ret := Filter(input, filter) diff --git a/libs/go-libs/logging/context.go b/libs/go-libs/logging/context.go index 7e8713231e..fcf9fb20c3 100644 --- a/libs/go-libs/logging/context.go +++ b/libs/go-libs/logging/context.go @@ -21,3 +21,17 @@ func FromContext(ctx context.Context) Logger { func ContextWithLogger(ctx context.Context, l Logger) context.Context { return context.WithValue(ctx, loggerKey, l) } + +func ContextWithFields(ctx context.Context, fields map[string]any) context.Context { + return ContextWithLogger(ctx, FromContext(ctx).WithFields(fields)) +} + +func ContextWithField(ctx context.Context, key string, value any) context.Context { + return ContextWithLogger(ctx, FromContext(ctx).WithFields(map[string]any{ + key: value, + })) +} + +func TestingContext() context.Context { + return ContextWithLogger(context.Background(), Testing()) +} diff --git a/libs/go-libs/logging/logging.go b/libs/go-libs/logging/logging.go index 495d9e5248..f161d462d9 100644 --- a/libs/go-libs/logging/logging.go +++ b/libs/go-libs/logging/logging.go @@ -10,6 +10,7 @@ type Logger interface { Info(args ...any) Error(args ...any) WithFields(map[string]any) Logger + WithField(key string, value any) Logger WithContext(ctx context.Context) Logger } diff --git a/libs/go-libs/logging/logrus.go b/libs/go-libs/logging/logrus.go index de01e87f8a..2ac5356f19 100644 --- a/libs/go-libs/logging/logrus.go +++ b/libs/go-libs/logging/logrus.go @@ -19,6 +19,7 @@ type logrusLogger struct { Errorf(format string, args ...any) Error(args ...any) WithFields(fields logrus.Fields) *logrus.Entry + WithField(key string, value any) *logrus.Entry WithContext(ctx context.Context) *logrus.Entry } } @@ -53,6 +54,12 @@ func (l *logrusLogger) WithFields(fields map[string]any) Logger { } } +func (l *logrusLogger) WithField(key string, value any) Logger { + return l.WithFields(map[string]any{ + key: value, + }) +} + var _ Logger = &logrusLogger{} func NewLogrus(logger *logrus.Logger) *logrusLogger { @@ -69,5 +76,11 @@ func Testing() *logrusLogger { logger.SetOutput(os.Stdout) logger.SetLevel(logrus.DebugLevel) } + + textFormatter := new(logrus.TextFormatter) + textFormatter.TimestampFormat = "15-01-2018 15:04:05.000000" + textFormatter.FullTimestamp = true + logger.SetFormatter(textFormatter) + return NewLogrus(logger) } diff --git a/libs/go-libs/pointer/utils.go b/libs/go-libs/pointer/utils.go new file mode 100644 index 0000000000..837c3991b0 --- /dev/null +++ b/libs/go-libs/pointer/utils.go @@ -0,0 +1,5 @@ +package pointer + +func For[T any](t T) *T { + return &t +} diff --git a/libs/go-libs/service/logging.go b/libs/go-libs/service/logging.go index 75bd0c7b30..5d308c80b7 100644 --- a/libs/go-libs/service/logging.go +++ b/libs/go-libs/service/logging.go @@ -14,15 +14,24 @@ import ( func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonFormattingLog bool) context.Context { l := logrus.New() l.SetOutput(w) - if debug { l.Level = logrus.DebugLevel } + var formatter logrus.Formatter if jsonFormattingLog { - l.SetFormatter(&logrus.JSONFormatter{}) + jsonFormatter := &logrus.JSONFormatter{} + jsonFormatter.TimestampFormat = "15-01-2018 15:04:05.000000" + formatter = jsonFormatter + } else { + textFormatter := new(logrus.TextFormatter) + textFormatter.TimestampFormat = "15-01-2018 15:04:05.000000" + textFormatter.FullTimestamp = true + formatter = textFormatter } + l.SetFormatter(formatter) + if viper.GetBool(otlptraces.OtelTracesFlag) { l.AddHook(otellogrus.NewHook(otellogrus.WithLevels( logrus.PanicLevel, diff --git a/tests/integration/go.mod b/tests/integration/go.mod index d9d9e79cfb..373c3aad41 100644 --- a/tests/integration/go.mod +++ b/tests/integration/go.mod @@ -46,6 +46,7 @@ require ( github.com/ThreeDotsLabs/watermill-kafka/v2 v2.2.2 // indirect github.com/ThreeDotsLabs/watermill-nats/v2 v2.0.0 // indirect github.com/ajg/form v1.5.1 // indirect + github.com/alitto/pond v1.8.3 // indirect github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/aquasecurity/esquery v0.2.0 // indirect github.com/bluele/gcache v0.0.2 // indirect diff --git a/tests/integration/go.sum b/tests/integration/go.sum index d17760a8b6..d7f6e456cc 100644 --- a/tests/integration/go.sum +++ b/tests/integration/go.sum @@ -413,6 +413,8 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alitto/pond v1.8.3 h1:ydIqygCLVPqIX/USe5EaV/aSRXTRXDEI9JwuDdu+/xs= +github.com/alitto/pond v1.8.3/go.mod h1:CmvIIGd5jKLasGI3D87qDkQxjzChdKMmnXMg3fG6M6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= diff --git a/tests/integration/internal/bootstrap.go b/tests/integration/internal/bootstrap.go index 1638041dc1..51173783b1 100644 --- a/tests/integration/internal/bootstrap.go +++ b/tests/integration/internal/bootstrap.go @@ -170,11 +170,13 @@ func startLedger() { "--publisher-nats-url=" + natsAddress(), fmt.Sprintf("--publisher-topic-mapping=*:%s-ledger", actualTestID), "--storage-postgres-conn-string=" + dsn.String(), + "--json-formatting-logger=false", "--bind=0.0.0.0:0", // Random port + "--debug", } - if testing.Verbose() { - args = append(args, "--debug") - } + //if testing.Verbose() { + // args = append(args, "--debug") + //} command.SetArgs(args) ledgerPort, ledgerCancel, ledgerErrCh = runAndWaitPort("ledger", command) } diff --git a/tests/integration/suite/ledger-balances.go b/tests/integration/suite/ledger-balances.go index aab43cd849..a4041e52ea 100644 --- a/tests/integration/suite/ledger-balances.go +++ b/tests/integration/suite/ledger-balances.go @@ -73,8 +73,8 @@ var _ = Given("some empty environment", func() { Expect(response.StatusCode).To(Equal(200)) Expect(response.BalancesCursorResponse.Cursor.Data).To(HaveLen(3)) Expect(response.BalancesCursorResponse.Cursor.Data[0]).To(Equal(map[string]map[string]*big.Int{ - "world": { - "USD": big.NewInt(-300), + "foo:bar": { + "USD": big.NewInt(200), }, })) Expect(response.BalancesCursorResponse.Cursor.Data[1]).To(Equal(map[string]map[string]*big.Int{ @@ -83,8 +83,8 @@ var _ = Given("some empty environment", func() { }, })) Expect(response.BalancesCursorResponse.Cursor.Data[2]).To(Equal(map[string]map[string]*big.Int{ - "foo:bar": { - "USD": big.NewInt(200), + "world": { + "USD": big.NewInt(-300), }, })) }) @@ -92,7 +92,7 @@ var _ = Given("some empty environment", func() { response, err := Client().Ledger.GetBalances( TestContext(), operations.GetBalancesRequest{ - Address: ptr("foo:.*"), + Address: ptr("foo:"), Ledger: "default", }, ) @@ -102,20 +102,20 @@ var _ = Given("some empty environment", func() { balancesCursorResponse := response.BalancesCursorResponse Expect(balancesCursorResponse.Cursor.Data).To(HaveLen(2)) Expect(balancesCursorResponse.Cursor.Data[0]).To(Equal(map[string]map[string]*big.Int{ - "foo:foo": { - "USD": big.NewInt(100), + "foo:bar": { + "USD": big.NewInt(200), }, })) Expect(balancesCursorResponse.Cursor.Data[1]).To(Equal(map[string]map[string]*big.Int{ - "foo:bar": { - "USD": big.NewInt(200), + "foo:foo": { + "USD": big.NewInt(100), }, })) response, err = Client().Ledger.GetBalances( TestContext(), operations.GetBalancesRequest{ - Address: ptr(".*:foo"), + Address: ptr(":foo"), Ledger: "default", }, ) @@ -149,7 +149,7 @@ var _ = Given("some empty environment", func() { response, err := Client().Ledger.GetBalancesAggregated( TestContext(), operations.GetBalancesAggregatedRequest{ - Address: ptr("foo:.*"), + Address: ptr("foo:"), Ledger: "default", }, ) @@ -240,6 +240,15 @@ var _ = Given("some environment with accounts and transactions", func() { }) } + balances = append([]map[string]map[string]*big.Int{ + { + "world": { + "USD": big.NewInt(-transactionCounts / 2 * 100), + "EUR": big.NewInt(-transactionCounts / 2 * 100), + }, + }, + }, balances...) + sort.Slice(balances, func(i, j int) bool { name1 := "" for name := range balances[i] { @@ -251,17 +260,8 @@ var _ = Given("some environment with accounts and transactions", func() { name2 = name break } - return name1 > name2 + return name1 < name2 }) - - balances = append([]map[string]map[string]*big.Int{ - { - "world": { - "USD": big.NewInt(-transactionCounts / 2 * 100), - "EUR": big.NewInt(-transactionCounts / 2 * 100), - }, - }, - }, balances...) }) AfterEach(func() { balances = nil diff --git a/tests/integration/suite/ledger-create-transaction.go b/tests/integration/suite/ledger-create-transaction.go index 670c8f42c6..9c5e4c010b 100644 --- a/tests/integration/suite/ledger-create-transaction.go +++ b/tests/integration/suite/ledger-create-transaction.go @@ -210,37 +210,6 @@ var _ = Given("some empty environment", func() { )) return true }).Should(BeTrue()) - - Eventually(func(g Gomega) bool { - response, err := Client().Search.Search( - TestContext(), - shared.Query{ - Target: ptr("ASSET"), - }, - ) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(response.StatusCode).To(Equal(200)) - - res := response.Response - g.Expect(res.Cursor.Data).To(HaveLen(2)) - g.Expect(res.Cursor.Data).To(ContainElements( - map[string]any{ - "account": "world", - "ledger": "default", - "output": float64(100), - "input": float64(0), - "name": "USD", - }, - map[string]any{ - "account": "alice", - "ledger": "default", - "output": float64(0), - "input": float64(100), - "name": "USD", - }, - )) - return true - }).Should(BeTrue()) }) }) }) diff --git a/tests/integration/suite/ledger-list-count-accounts.go b/tests/integration/suite/ledger-list-count-accounts.go index 9b2fd9e3cd..4313e38b79 100644 --- a/tests/integration/suite/ledger-list-count-accounts.go +++ b/tests/integration/suite/ledger-list-count-accounts.go @@ -2,7 +2,7 @@ package suite import ( "fmt" - "math/big" + "sort" "time" "github.com/formancehq/formance-sdk-go/pkg/models/operations" @@ -96,14 +96,13 @@ var _ = Given("some empty environment", func() { accountsCursorResponse := response.AccountsCursorResponse Expect(accountsCursorResponse.Cursor.Data).To(HaveLen(3)) Expect(accountsCursorResponse.Cursor.Data[0]).To(Equal(shared.Account{ - Address: "foo:foo", - Metadata: metadata1, - })) - Expect(accountsCursorResponse.Cursor.Data[1]).To(Equal(shared.Account{ Address: "foo:bar", Metadata: metadata2, })) - + Expect(accountsCursorResponse.Cursor.Data[1]).To(Equal(shared.Account{ + Address: "foo:foo", + Metadata: metadata1, + })) Expect(accountsCursorResponse.Cursor.Data[2]).To(Equal(shared.Account{ Address: "world", Metadata: metadata.Metadata{}, @@ -113,7 +112,7 @@ var _ = Given("some empty environment", func() { response, err := Client().Ledger.ListAccounts( TestContext(), operations.ListAccountsRequest{ - Address: ptr("foo:.*"), + Address: ptr("foo:"), Ledger: "default", }, ) @@ -123,59 +122,18 @@ var _ = Given("some empty environment", func() { accountsCursorResponse := response.AccountsCursorResponse Expect(accountsCursorResponse.Cursor.Data).To(HaveLen(2)) Expect(accountsCursorResponse.Cursor.Data[0]).To(Equal(shared.Account{ - Address: "foo:foo", - Metadata: metadata1, - })) - Expect(accountsCursorResponse.Cursor.Data[1]).To(Equal(shared.Account{ Address: "foo:bar", Metadata: metadata2, })) - - response, err = Client().Ledger.ListAccounts( - TestContext(), - operations.ListAccountsRequest{ - Address: ptr(".*:foo"), - Ledger: "default", - }, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(200)) - - accountsCursorResponse = response.AccountsCursorResponse - Expect(accountsCursorResponse.Cursor.Data).To(HaveLen(1)) - Expect(accountsCursorResponse.Cursor.Data[0]).To(Equal(shared.Account{ + Expect(accountsCursorResponse.Cursor.Data[1]).To(Equal(shared.Account{ Address: "foo:foo", Metadata: metadata1, })) - }) - It("should be listed on api using balance filters", func() { - response, err := Client().Ledger.ListAccounts( - TestContext(), - operations.ListAccountsRequest{ - Balance: big.NewInt(90), - BalanceOperator: ptr(operations.ListAccountsBalanceOperatorLte), - Ledger: "default", - }, - ) - Expect(err).ToNot(HaveOccurred()) - Expect(response.StatusCode).To(Equal(200)) - - accountsCursorResponse := response.AccountsCursorResponse - Expect(accountsCursorResponse.Cursor.Data).To(HaveLen(2)) - Expect(accountsCursorResponse.Cursor.Data[0]).To(Equal(shared.Account{ - Address: "foo:bar", - Metadata: metadata2, - })) - Expect(accountsCursorResponse.Cursor.Data[1]).To(Equal(shared.Account{ - Address: "world", - Metadata: metadata.Metadata{}, - })) - // Default operator should be gte response, err = Client().Ledger.ListAccounts( TestContext(), operations.ListAccountsRequest{ - Balance: big.NewInt(90), + Address: ptr(":foo"), Ledger: "default", }, ) @@ -272,6 +230,10 @@ var _ = Given("some environment with accounts", func() { Address: fmt.Sprintf("foo:%d", i), Metadata: m, }) + + sort.Slice(accounts, func(i, j int) bool { + return accounts[i].Address < accounts[j].Address + }) } }) AfterEach(func() { From cb1ecea3a58310a6ed5cafe231b8c2bd330b5ab0 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 13 Jul 2023 15:33:31 +0200 Subject: [PATCH 2/7] feat: reuse batching worker for logs ingestion --- components/ledger/pkg/bus/monitor.go | 4 +- components/ledger/pkg/core/log.go | 30 ----- .../ledger/pkg/ledger/command/commander.go | 39 +++--- .../pkg/ledger/command/commander_test.go | 85 ++++++++++---- .../ledger/pkg/ledger/command/context.go | 27 +++-- components/ledger/pkg/ledger/command/store.go | 3 +- components/ledger/pkg/ledger/ledger.go | 64 ++++++---- .../ledger/pkg/ledger/query/projector.go | 41 ++++--- .../ledger/pkg/ledger/query/projector_test.go | 4 +- components/ledger/pkg/ledger/resolver.go | 24 +--- .../pkg/ledger/utils/batching/batcher.go | 46 ++++++-- components/ledger/pkg/storage/driver/cli.go | 9 +- .../ledger/pkg/storage/driver/driver.go | 11 +- .../ledger/pkg/storage/ledgerstore/logs.go | 2 +- .../pkg/storage/ledgerstore/logs_test.go | 4 +- .../pkg/storage/ledgerstore/logs_worker.go | 111 ------------------ .../pkg/storage/ledgerstore/main_test.go | 7 +- .../ledgerstore/migrates/0-init-schema/any.go | 1 - .../ledger/pkg/storage/ledgerstore/store.go | 47 +------- .../pkg/storage/ledgerstore/transactions.go | 1 - .../pkg/storage/storagetesting/storage.go | 3 +- components/ledger/pkg/storage/tx.go | 1 - 22 files changed, 222 insertions(+), 342 deletions(-) delete mode 100644 components/ledger/pkg/storage/ledgerstore/logs_worker.go diff --git a/components/ledger/pkg/bus/monitor.go b/components/ledger/pkg/bus/monitor.go index 19a76c32dd..b04c1b2bda 100644 --- a/components/ledger/pkg/bus/monitor.go +++ b/components/ledger/pkg/bus/monitor.go @@ -12,7 +12,7 @@ import ( ) type ledgerMonitor struct { - publisher message.Publisher + publisher message.Publisher ledgerName string } @@ -20,7 +20,7 @@ var _ query.Monitor = &ledgerMonitor{} func NewLedgerMonitor(publisher message.Publisher, ledgerName string) *ledgerMonitor { m := &ledgerMonitor{ - publisher: publisher, + publisher: publisher, ledgerName: ledgerName, } return m diff --git a/components/ledger/pkg/core/log.go b/components/ledger/pkg/core/log.go index 5ecb4998ff..9c830f6969 100644 --- a/components/ledger/pkg/core/log.go +++ b/components/ledger/pkg/core/log.go @@ -389,33 +389,3 @@ var ( }, } ) - -type LogPersistenceTracker struct { - activeLog *ActiveLog - done chan struct{} -} - -func (r *LogPersistenceTracker) ActiveLog() *ActiveLog { - return r.activeLog -} - -func (r *LogPersistenceTracker) Resolve() { - close(r.done) -} - -func (r *LogPersistenceTracker) Done() chan struct{} { - return r.done -} - -func NewLogPersistenceTracker(log *ActiveLog) *LogPersistenceTracker { - return &LogPersistenceTracker{ - activeLog: log, - done: make(chan struct{}), - } -} - -func NewResolvedLogPersistenceTracker(log *ActiveLog) *LogPersistenceTracker { - ret := NewLogPersistenceTracker(log) - ret.Resolve() - return ret -} diff --git a/components/ledger/pkg/ledger/command/commander.go b/components/ledger/pkg/ledger/command/commander.go index 3a4d3b5fc5..5433b5c62c 100644 --- a/components/ledger/pkg/ledger/command/commander.go +++ b/components/ledger/pkg/ledger/command/commander.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "github.com/formancehq/ledger/pkg/core" + "github.com/formancehq/ledger/pkg/ledger/utils/batching" "github.com/formancehq/ledger/pkg/machine" "github.com/formancehq/ledger/pkg/machine/vm" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" @@ -24,6 +25,7 @@ type Parameters struct { } type Commander struct { + *batching.Batcher[*core.ActiveLog] store Store locker Locker metricsRegistry metrics.PerLedgerRegistry @@ -42,6 +44,7 @@ func New( compiler *Compiler, referencer *Referencer, metricsRegistry metrics.PerLedgerRegistry, + onBatchProcessed batching.OnBatchProcessed[*core.ActiveLog], ) *Commander { log, err := store.ReadLastLogWithType(context.Background(), core.NewTransactionLogType, core.RevertedTransactionLogType) if err != nil && !storageerrors.IsNotFoundError(err) { @@ -79,6 +82,7 @@ func New( referencer: referencer, lastTXID: lastTXID, lastLog: lastLog, + Batcher: batching.NewBatcher(store.InsertLogs, onBatchProcessed, 1, 4096), } } @@ -98,37 +102,37 @@ func (commander *Commander) exec(ctx context.Context, parameters Parameters, scr } execContext := newExecutionContext(commander, parameters) - tracker, err := execContext.run(ctx, func(executionContext *executionContext) (*core.LogPersistenceTracker, error) { + return execContext.run(ctx, func(executionContext *executionContext) (*core.ActiveLog, chan struct{}, error) { if script.Reference != "" { if err := commander.referencer.take(referenceTxReference, script.Reference); err != nil { - return nil, ErrConflictError + return nil, nil, ErrConflictError } defer commander.referencer.release(referenceTxReference, script.Reference) _, err := commander.store.ReadLogForCreatedTransactionWithReference(ctx, script.Reference) if err == nil { - return nil, ErrConflictError + return nil, nil, ErrConflictError } if err != storageerrors.ErrNotFound && err != nil { - return nil, err + return nil, nil, err } } program, err := commander.compiler.Compile(ctx, script.Plain) if err != nil { - return nil, errorsutil.NewError(ErrCompilationFailed, errors.Wrap(err, "compiling numscript")) + return nil, nil, errorsutil.NewError(ErrCompilationFailed, errors.Wrap(err, "compiling numscript")) } m := vm.NewMachine(*program) if err := m.SetVarsFromJSON(script.Vars); err != nil { - return nil, errorsutil.NewError(ErrCompilationFailed, + return nil, nil, errorsutil.NewError(ErrCompilationFailed, errors.Wrap(err, "could not set variables")) } involvedAccounts, involvedSources, err := m.ResolveResources(ctx, commander.store) if err != nil { - return nil, errorsutil.NewError(ErrCompilationFailed, + return nil, nil, errorsutil.NewError(ErrCompilationFailed, errors.Wrap(err, "could not resolve program resources")) } @@ -140,23 +144,23 @@ func (commander *Commander) exec(ctx context.Context, parameters Parameters, scr unlock, err := commander.locker.Lock(ctx, lockAccounts) if err != nil { - return nil, errors.Wrap(err, "locking accounts for tx processing") + return nil, nil, errors.Wrap(err, "locking accounts for tx processing") } unlock(ctx) err = m.ResolveBalances(ctx, commander.store) if err != nil { - return nil, errorsutil.NewError(ErrCompilationFailed, + return nil, nil, errorsutil.NewError(ErrCompilationFailed, errors.Wrap(err, "could not resolve balances")) } result, err := machine.Run(m, script) if err != nil { - return nil, errors.Wrap(err, "running numscript") + return nil, nil, errors.Wrap(err, "running numscript") } if len(result.Postings) == 0 { - return nil, ErrNoPostings + return nil, nil, ErrNoPostings } tx := core.NewTransaction(). @@ -173,10 +177,6 @@ func (commander *Commander) exec(ctx context.Context, parameters Parameters, scr return executionContext.AppendLog(ctx, log) }) - if err != nil { - return nil, err - } - return tracker.ActiveLog().ChainedLog, nil } func (commander *Commander) CreateTransaction(ctx context.Context, parameters Parameters, script core.RunScript) (*core.Transaction, error) { @@ -200,7 +200,7 @@ func (commander *Commander) SaveMeta(ctx context.Context, parameters Parameters, } execContext := newExecutionContext(commander, parameters) - _, err := execContext.run(ctx, func(executionContext *executionContext) (*core.LogPersistenceTracker, error) { + _, err := execContext.run(ctx, func(executionContext *executionContext) (*core.ActiveLog, chan struct{}, error) { var ( log *core.Log at = core.Now() @@ -209,7 +209,7 @@ func (commander *Commander) SaveMeta(ctx context.Context, parameters Parameters, case core.MetaTargetTypeTransaction: _, err := commander.store.ReadLogForCreatedTransaction(ctx, targetID.(uint64)) if err != nil { - return nil, err + return nil, nil, err } log = core.NewSetMetadataLog(at, core.SetMetadataLogPayload{ TargetType: core.MetaTargetTypeTransaction, @@ -223,7 +223,7 @@ func (commander *Commander) SaveMeta(ctx context.Context, parameters Parameters, Metadata: m, }) default: - return nil, errorsutil.NewError(ErrValidation, errors.Errorf("unknown target type '%s'", targetType)) + return nil, nil, errorsutil.NewError(ErrValidation, errors.Errorf("unknown target type '%s'", targetType)) } return executionContext.AppendLog(ctx, log) @@ -274,7 +274,8 @@ func (commander *Commander) RevertTransaction(ctx context.Context, parameters Pa return log.Data.(core.RevertedTransactionLogPayload).RevertTransaction, nil } -func (commander *Commander) Wait() { +func (commander *Commander) Close() { + commander.Batcher.Close() commander.running.Wait() } diff --git a/components/ledger/pkg/ledger/command/commander_test.go b/components/ledger/pkg/ledger/command/commander_test.go index 85a025e747..547ce545ea 100644 --- a/components/ledger/pkg/ledger/command/commander_test.go +++ b/components/ledger/pkg/ledger/command/commander_test.go @@ -8,6 +8,7 @@ import ( "github.com/formancehq/ledger/pkg/core" storageerrors "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/stack/libs/go-libs/collectionutils" + "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/pkg/errors" "github.com/stretchr/testify/require" @@ -129,21 +130,18 @@ func (m *mockStore) ReadLastLogWithType(background context.Context, logType ...c return first, nil } -func (m *mockStore) AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) { - var ( - previous, persistedLog *core.ChainedLog - ) - if len(m.logs) > 0 { - previous = m.logs[len(m.logs)-1] - } - persistedLog = log.ChainLog(previous) - m.logs = append(m.logs, persistedLog) +func (m *mockStore) InsertLogs(ctx context.Context, logs ...*core.ActiveLog) error { - ret := core.NewLogPersistenceTracker(log) - ret.Resolve() - log.SetProjected() + for _, log := range logs { + var previousLog *core.ChainedLog + if len(m.logs) > 0 { + previousLog = m.logs[len(m.logs)-1] + } + chainedLog := log.ChainLog(previousLog) + m.logs = append(m.logs, chainedLog) + } - return ret, nil + return nil } var ( @@ -204,7 +202,7 @@ var testCases = []testCase{ WithPostings(core.NewPosting("world", "mint", "GEM", big.NewInt(100))). WithReference("tx_ref") log := core.NewTransactionLog(tx, nil) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) + err := store.InsertLogs(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) }, script: ` @@ -269,7 +267,7 @@ var testCases = []testCase{ WithTimestamp(now), map[string]metadata.Metadata{}, ).WithIdempotencyKey("testing") - _, err := r.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) + err := r.InsertLogs(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) }, parameters: Parameters{ @@ -286,13 +284,21 @@ func TestCreateTransaction(t *testing.T) { t.Run(tc.name, func(t *testing.T) { store := newMockStore() + ctx := logging.TestingContext() - ledger := New(store, NoOpLocker, NewCompiler(1024), NewReferencer(), nil) + commander := New(store, NoOpLocker, NewCompiler(1024), + NewReferencer(), nil, func(activeLogs ...*core.ActiveLog) { + for _, activeLog := range activeLogs { + activeLog.SetProjected() + } + }) + go commander.Run(ctx) + defer commander.Close() if tc.setup != nil { tc.setup(t, store) } - ret, err := ledger.CreateTransaction(context.Background(), tc.parameters, core.RunScript{ + ret, err := commander.CreateTransaction(ctx, tc.parameters, core.RunScript{ Script: core.Script{ Plain: tc.script, }, @@ -326,50 +332,77 @@ func TestCreateTransaction(t *testing.T) { func TestRevert(t *testing.T) { txID := uint64(0) store := newMockStore() + ctx := logging.TestingContext() + log := core.NewTransactionLog( core.NewTransaction().WithPostings( core.NewPosting("world", "bank", "USD", big.NewInt(100)), ), map[string]metadata.Metadata{}, ) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) + err := store.InsertLogs(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) - ledger := New(store, NoOpLocker, NewCompiler(1024), NewReferencer(), nil) - _, err = ledger.RevertTransaction(context.Background(), Parameters{}, txID) + commander := New(store, NoOpLocker, NewCompiler(1024), NewReferencer(), nil, func(activeLogs ...*core.ActiveLog) { + for _, activeLog := range activeLogs { + activeLog.SetProjected() + } + }) + go commander.Run(ctx) + defer commander.Close() + + _, err = commander.RevertTransaction(ctx, Parameters{}, txID) require.NoError(t, err) } func TestRevertWithAlreadyReverted(t *testing.T) { store := newMockStore() + ctx := logging.TestingContext() + log := core. NewRevertedTransactionLog(core.Now(), 0, core.NewTransaction()) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) + err := store.InsertLogs(context.Background(), core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) - ledger := New(store, NoOpLocker, NewCompiler(1024), NewReferencer(), nil) + commander := New(store, NoOpLocker, NewCompiler(1024), NewReferencer(), nil, func(activeLogs ...*core.ActiveLog) { + for _, activeLog := range activeLogs { + activeLog.SetProjected() + } + }) + go commander.Run(ctx) + defer commander.Close() - _, err = ledger.RevertTransaction(context.Background(), Parameters{}, 0) + _, err = commander.RevertTransaction(context.Background(), Parameters{}, 0) require.True(t, errors.Is(err, ErrAlreadyReverted)) } func TestRevertWithRevertOccurring(t *testing.T) { store := newMockStore() + ctx := logging.TestingContext() + log := core.NewTransactionLog( core.NewTransaction().WithPostings( core.NewPosting("world", "bank", "USD", big.NewInt(100)), ), map[string]metadata.Metadata{}, ) - _, err := store.AppendLog(context.Background(), core.NewActiveLog(log.ChainLog(nil))) + err := store.InsertLogs(ctx, core.NewActiveLog(log.ChainLog(nil))) require.NoError(t, err) referencer := NewReferencer() - ledger := New(store, NoOpLocker, NewCompiler(1024), referencer, nil) + commander := New(store, NoOpLocker, NewCompiler(1024), + referencer, nil, func(activeLogs ...*core.ActiveLog) { + for _, activeLog := range activeLogs { + activeLog.SetProjected() + } + }) + go commander.Run(ctx) + defer commander.Close() + referencer.take(referenceReverts, uint64(0)) - _, err = ledger.RevertTransaction(context.Background(), Parameters{}, 0) + _, err = commander.RevertTransaction(ctx, Parameters{}, 0) require.True(t, errors.Is(err, ErrRevertOccurring)) } diff --git a/components/ledger/pkg/ledger/command/context.go b/components/ledger/pkg/ledger/command/context.go index 1ec3c5d1ba..98363ca2b8 100644 --- a/components/ledger/pkg/ledger/command/context.go +++ b/components/ledger/pkg/ledger/command/context.go @@ -13,20 +13,25 @@ type executionContext struct { parameters Parameters } -func (e *executionContext) AppendLog(ctx context.Context, log *core.Log) (*core.LogPersistenceTracker, error) { +func (e *executionContext) AppendLog(ctx context.Context, log *core.Log) (*core.ActiveLog, chan struct{}, error) { if e.parameters.DryRun { - chainedLog := log.ChainLog(nil) - return core.NewResolvedLogPersistenceTracker(core.NewActiveLog(chainedLog)), nil + ret := make(chan struct{}) + close(ret) + return core.NewActiveLog(log.ChainLog(nil)), ret, nil } activeLog := core.NewActiveLog(e.commander.chainLog(log)) logging.FromContext(ctx).WithFields(map[string]any{ "id": activeLog.ChainedLog.ID, }).Debugf("Appending log") - return e.commander.store.AppendLog(ctx, activeLog) + done := make(chan struct{}) + e.commander.Append(activeLog, func() { + close(done) + }) + return activeLog, done, nil } -func (e *executionContext) run(ctx context.Context, executor func(e *executionContext) (*core.LogPersistenceTracker, error)) (*core.LogPersistenceTracker, error) { +func (e *executionContext) run(ctx context.Context, executor func(e *executionContext) (*core.ActiveLog, chan struct{}, error)) (*core.ChainedLog, error) { if ik := e.parameters.IdempotencyKey; ik != "" { if err := e.commander.referencer.take(referenceIks, ik); err != nil { return nil, err @@ -35,26 +40,26 @@ func (e *executionContext) run(ctx context.Context, executor func(e *executionCo chainedLog, err := e.commander.store.ReadLogWithIdempotencyKey(ctx, ik) if err == nil { - return core.NewResolvedLogPersistenceTracker(core.NewActiveLog(chainedLog)), nil + return chainedLog, nil } if err != storageerrors.ErrNotFound && err != nil { return nil, err } } - tracker, err := executor(e) + activeLog, done, err := executor(e) if err != nil { return nil, err } - <-tracker.Done() + <-done logger := logging.FromContext(ctx).WithFields(map[string]any{ - "id": tracker.ActiveLog().ChainedLog.ID, + "id": activeLog.ID, }) logger.Debugf("Log inserted in database") if !e.parameters.Async { - <-tracker.ActiveLog().Projected + <-activeLog.Projected logger.Debugf("Log fully ingested") } - return tracker, nil + return activeLog.ChainedLog, nil } func newExecutionContext(commander *Commander, parameters Parameters) *executionContext { diff --git a/components/ledger/pkg/ledger/command/store.go b/components/ledger/pkg/ledger/command/store.go index c08f3a7b43..3a020048b5 100644 --- a/components/ledger/pkg/ledger/command/store.go +++ b/components/ledger/pkg/ledger/command/store.go @@ -9,7 +9,8 @@ import ( type Store interface { vm.Store - AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) + //AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) + InsertLogs(ctx context.Context, logs ...*core.ActiveLog) error ReadLastLogWithType(ctx context.Context, logType ...core.LogType) (*core.ChainedLog, error) ReadLogForCreatedTransactionWithReference(ctx context.Context, reference string) (*core.ChainedLog, error) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) (*core.ChainedLog, error) diff --git a/components/ledger/pkg/ledger/ledger.go b/components/ledger/pkg/ledger/ledger.go index fd661930b2..4fdd1ec27f 100644 --- a/components/ledger/pkg/ledger/ledger.go +++ b/components/ledger/pkg/ledger/ledger.go @@ -3,6 +3,8 @@ package ledger import ( "context" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/formancehq/ledger/pkg/bus" "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/ledger/command" "github.com/formancehq/ledger/pkg/ledger/query" @@ -10,50 +12,68 @@ import ( "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/api" "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/formancehq/stack/libs/go-libs/metadata" "github.com/pkg/errors" ) type Ledger struct { - *command.Commander + commander *command.Commander store *ledgerstore.Store projector *query.Projector - locker *command.DefaultLocker +} + +func (l *Ledger) CreateTransaction(ctx context.Context, parameters command.Parameters, data core.RunScript) (*core.Transaction, error) { + return l.commander.CreateTransaction(ctx, parameters, data) +} + +func (l *Ledger) RevertTransaction(ctx context.Context, parameters command.Parameters, id uint64) (*core.Transaction, error) { + return l.commander.RevertTransaction(ctx, parameters, id) +} + +func (l *Ledger) SaveMeta(ctx context.Context, parameters command.Parameters, targetType string, targetID any, m metadata.Metadata) error { + return l.commander.SaveMeta(ctx, parameters, targetType, targetID, m) } func New( + name string, store *ledgerstore.Store, - locker *command.DefaultLocker, - queryWorker *query.Projector, + publisher message.Publisher, compiler *command.Compiler, - metricsRegistry metrics.PerLedgerRegistry, ) *Ledger { - store.OnLogWrote(func(logs []*core.ActiveLog) { - if err := queryWorker.QueueLog(logs...); err != nil { - panic(err) - } - }) + var monitor query.Monitor = query.NewNoOpMonitor() + if publisher != nil { + monitor = bus.NewLedgerMonitor(publisher, name) + } + metricsRegistry, err := metrics.RegisterPerLedgerMetricsRegistry(name) + if err != nil { + panic(err) + } + projector := query.NewProjector(store, monitor, metricsRegistry) return &Ledger{ - Commander: command.New(store, locker, compiler, command.NewReferencer(), metricsRegistry), + commander: command.New( + store, + command.NewDefaultLocker(), + compiler, + command.NewReferencer(), + metricsRegistry, + projector.QueueLog, + ), store: store, - projector: queryWorker, - locker: locker, + projector: projector, } } -func (l *Ledger) Close(ctx context.Context) error { +func (l *Ledger) Start(ctx context.Context) { + go l.commander.Run(logging.ContextWithField(ctx, "component", "commander")) + l.projector.Start(logging.ContextWithField(ctx, "component", "projector")) +} +func (l *Ledger) Close(ctx context.Context) { logging.FromContext(ctx).Debugf("Close commander") - l.Commander.Wait() - - logging.FromContext(ctx).Debugf("Close storage worker") - if err := l.store.Stop(logging.ContextWithField(ctx, "component", "store")); err != nil { - return errors.Wrap(err, "stopping ledger store") - } + l.commander.Close() logging.FromContext(ctx).Debugf("Close projector") l.projector.Stop(logging.ContextWithField(ctx, "component", "projector")) - - return nil } func (l *Ledger) GetTransactions(ctx context.Context, q ledgerstore.TransactionsQuery) (*api.Cursor[core.ExpandedTransaction], error) { diff --git a/components/ledger/pkg/ledger/query/projector.go b/components/ledger/pkg/ledger/query/projector.go index c6e583d9df..197c506f59 100644 --- a/components/ledger/pkg/ledger/query/projector.go +++ b/components/ledger/pkg/ledger/query/projector.go @@ -61,10 +61,8 @@ type Projector struct { limitReadLogs int } -func (p *Projector) QueueLog(logs ...*core.ActiveLog) error { +func (p *Projector) QueueLog(logs ...*core.ActiveLog) { p.queue <- logs - - return nil } func (p *Projector) Stop(ctx context.Context) { @@ -245,16 +243,31 @@ func NewProjector( metricsRegistry metrics.PerLedgerRegistry, ) *Projector { return &Projector{ - store: store, - monitor: monitor, - metricsRegistry: metricsRegistry, - txWorker: batching.NewBatcher(store.InsertTransactions, 2, 512), - accountMetadataWorker: batching.NewBatcher(store.UpdateAccountsMetadata, 1, 512), - txMetadataWorker: batching.NewBatcher(store.UpdateTransactionsMetadata, 1, 512), - moveBuffer: newMoveBuffer(store.InsertMoves, 5, 100), - activeLogs: collectionutils.NewLinkedList[*core.ActiveLog](), - queue: make(chan []*core.ActiveLog, 1024), - stopChan: make(chan chan struct{}), - limitReadLogs: 10000, + store: store, + monitor: monitor, + metricsRegistry: metricsRegistry, + txWorker: batching.NewBatcher( + store.InsertTransactions, + batching.NoOpOnBatchProcessed[core.Transaction](), + 2, + 512, + ), + accountMetadataWorker: batching.NewBatcher( + store.UpdateAccountsMetadata, + batching.NoOpOnBatchProcessed[core.Account](), + 1, + 512, + ), + txMetadataWorker: batching.NewBatcher( + store.UpdateTransactionsMetadata, + batching.NoOpOnBatchProcessed[core.TransactionWithMetadata](), + 1, + 512, + ), + moveBuffer: newMoveBuffer(store.InsertMoves, 5, 100), + activeLogs: collectionutils.NewLinkedList[*core.ActiveLog](), + queue: make(chan []*core.ActiveLog, 1024), + stopChan: make(chan chan struct{}), + limitReadLogs: 10000, } } diff --git a/components/ledger/pkg/ledger/query/projector_test.go b/components/ledger/pkg/ledger/query/projector_test.go index a7bbb87b0a..d12bb5a77d 100644 --- a/components/ledger/pkg/ledger/query/projector_test.go +++ b/components/ledger/pkg/ledger/query/projector_test.go @@ -65,7 +65,7 @@ func TestProjector(t *testing.T) { for i, chainedLog := range logs { chainedLog.ID = uint64(i) activeLog := core.NewActiveLog(chainedLog) - require.NoError(t, projector.QueueLog(activeLog)) + projector.QueueLog(activeLog) <-activeLog.Projected } @@ -166,7 +166,7 @@ func TestProjectorUnderHeavyParallelLoad(t *testing.T) { ), nil).ChainLog(previousLog) activeLog := core.NewActiveLog(log) pool.Submit(func() { - require.NoError(t, projector.QueueLog(activeLog)) + projector.QueueLog(activeLog) }) previousLog = log allLogs = append(allLogs, activeLog) diff --git a/components/ledger/pkg/ledger/resolver.go b/components/ledger/pkg/ledger/resolver.go index b00527ecfc..24d426f91b 100644 --- a/components/ledger/pkg/ledger/resolver.go +++ b/components/ledger/pkg/ledger/resolver.go @@ -5,9 +5,7 @@ import ( "sync" "github.com/ThreeDotsLabs/watermill/message" - "github.com/formancehq/ledger/pkg/bus" "github.com/formancehq/ledger/pkg/ledger/command" - "github.com/formancehq/ledger/pkg/ledger/query" "github.com/formancehq/ledger/pkg/opentelemetry/metrics" "github.com/formancehq/ledger/pkg/storage/driver" "github.com/formancehq/ledger/pkg/storage/ledgerstore" @@ -107,22 +105,8 @@ func (r *Resolver) GetLedger(ctx context.Context, name string) (*Ledger, error) } } - locker := command.NewDefaultLocker() - - metricsRegistry, err := metrics.RegisterPerLedgerMetricsRegistry(name) - if err != nil { - return nil, errors.Wrap(err, "registering metrics") - } - - var monitor query.Monitor = query.NewNoOpMonitor() - if r.publisher != nil { - monitor = bus.NewLedgerMonitor(r.publisher, name) - } - - projector := query.NewProjector(store, monitor, metricsRegistry) - projector.Start(logging.ContextWithLogger(context.Background(), r.logger)) - - ledger = New(store, locker, projector, r.compiler, metricsRegistry) + ledger = New(name, store, r.publisher, r.compiler) + ledger.Start(logging.ContextWithLogger(context.Background(), r.logger)) r.ledgers[name] = ledger r.metricsRegistry.ActiveLedgers().Add(ctx, +1) } @@ -137,9 +121,7 @@ func (r *Resolver) CloseLedgers(ctx context.Context) error { }() for name, ledger := range r.ledgers { r.logger.Infof("Close ledger %s", name) - if err := ledger.Close(logging.ContextWithLogger(ctx, r.logger.WithField("ledger", name))); err != nil { - return err - } + ledger.Close(logging.ContextWithLogger(ctx, r.logger.WithField("ledger", name))) delete(r.ledgers, name) } diff --git a/components/ledger/pkg/ledger/utils/batching/batcher.go b/components/ledger/pkg/ledger/utils/batching/batcher.go index 0ff54cbde5..ec435d7b07 100644 --- a/components/ledger/pkg/ledger/utils/batching/batcher.go +++ b/components/ledger/pkg/ledger/utils/batching/batcher.go @@ -9,28 +9,43 @@ import ( "github.com/formancehq/stack/libs/go-libs/collectionutils" ) +type OnBatchProcessed[T any] func(...T) + +func NoOpOnBatchProcessed[T any]() func(...T) { + return func(t ...T) {} +} + type pending[T any] struct { object T callback func() } -type batcherJob[T any] []*pending[T] +type batcherJob[T any] struct { + items []*pending[T] + onBatchProcessed OnBatchProcessed[T] +} func (b batcherJob[T]) String() string { - return fmt.Sprintf("processing %d items", len(b)) + return fmt.Sprintf("processing %d items", len(b.items)) } func (b batcherJob[T]) Terminated() { - for _, v := range b { + for _, v := range b.items { v.callback() } + if b.onBatchProcessed != nil { + b.onBatchProcessed(collectionutils.Map(b.items, func(from *pending[T]) T { + return from.object + })...) + } } type Batcher[T any] struct { *job.Runner[batcherJob[T]] - pending []*pending[T] - mu sync.Mutex - maxBatchSize int + pending []*pending[T] + mu sync.Mutex + maxBatchSize int + onBatchProcessed OnBatchProcessed[T] } func (s *Batcher[T]) Append(object T, callback func()) { @@ -53,21 +68,26 @@ func (s *Batcher[T]) nextBatch() *batcherJob[T] { if len(s.pending) > s.maxBatchSize { batch := s.pending[:s.maxBatchSize] s.pending = s.pending[s.maxBatchSize:] - ret := batcherJob[T](batch) - return &ret + return &batcherJob[T]{ + onBatchProcessed: s.onBatchProcessed, + items: batch, + } } batch := s.pending s.pending = make([]*pending[T], 0) - ret := batcherJob[T](batch) - return &ret + return &batcherJob[T]{ + items: batch, + onBatchProcessed: s.onBatchProcessed, + } } -func NewBatcher[T any](runner func(context.Context, ...T) error, nbWorkers, maxBatchSize int) *Batcher[T] { +func NewBatcher[T any](runner func(context.Context, ...T) error, onBatchProcessed OnBatchProcessed[T], nbWorkers, maxBatchSize int) *Batcher[T] { ret := &Batcher[T]{ - maxBatchSize: maxBatchSize, + maxBatchSize: maxBatchSize, + onBatchProcessed: onBatchProcessed, } ret.Runner = job.NewJobRunner[batcherJob[T]](func(ctx context.Context, job *batcherJob[T]) error { - return runner(ctx, collectionutils.Map(*job, func(from *pending[T]) T { + return runner(ctx, collectionutils.Map(job.items, func(from *pending[T]) T { return from.object })...) }, ret.nextBatch, nbWorkers) diff --git a/components/ledger/pkg/storage/driver/cli.go b/components/ledger/pkg/storage/driver/cli.go index f7bdcae0cd..f2d3e1d8f1 100644 --- a/components/ledger/pkg/storage/driver/cli.go +++ b/components/ledger/pkg/storage/driver/cli.go @@ -5,7 +5,6 @@ import ( "time" "github.com/formancehq/ledger/pkg/storage" - "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/health" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -40,7 +39,6 @@ type PostgresConfig struct { type ModuleConfig struct { PostgresConnectionOptions storage.ConnectionOptions - StoreConfig ledgerstore.StoreConfig Debug bool } @@ -61,12 +59,7 @@ func CLIModule(v *viper.Viper, output io.Writer, debug bool) fx.Option { return storage.NewDatabase(db) })) options = append(options, fx.Provide(func(db *storage.Database) (*Driver, error) { - return New("postgres", db, ledgerstore.StoreConfig{ - StoreWorkerConfig: ledgerstore.Config{ - MaxPendingSize: v.GetInt(StoreWorkerMaxPendingSize), - MaxWriteChanSize: v.GetInt(StoreWorkerMaxWriteChanSize), - }, - }), nil + return New("postgres", db), nil })) options = append(options, health.ProvideHealthCheck(func(db *bun.DB) health.NamedCheck { return health.NewNamedCheck("postgres", health.CheckFn(db.PingContext)) diff --git a/components/ledger/pkg/storage/driver/driver.go b/components/ledger/pkg/storage/driver/driver.go index 6edae8c876..ba4ee7d9e4 100644 --- a/components/ledger/pkg/storage/driver/driver.go +++ b/components/ledger/pkg/storage/driver/driver.go @@ -69,7 +69,6 @@ type Driver struct { db *storage.Database systemStore *systemstore.Store lock sync.Mutex - storeConfig ledgerstore.StoreConfig } func (d *Driver) GetSystemStore() *systemstore.Store { @@ -88,11 +87,10 @@ func (d *Driver) newStore(ctx context.Context, name string) (*ledgerstore.Store, store, err := ledgerstore.New(schema, func(ctx context.Context) error { return d.GetSystemStore().DeleteLedger(ctx, name) - }, d.storeConfig) + }) if err != nil { return nil, err } - go store.Run(logging.ContextWithLogger(context.Background(), logging.FromContext(ctx).WithField("component", "store"))) return store, nil } @@ -175,10 +173,9 @@ func (d *Driver) Close(ctx context.Context) error { return d.db.Close(ctx) } -func New(name string, db *storage.Database, storeConfig ledgerstore.StoreConfig) *Driver { +func New(name string, db *storage.Database) *Driver { return &Driver{ - db: db, - name: name, - storeConfig: storeConfig, + db: db, + name: name, } } diff --git a/components/ledger/pkg/storage/ledgerstore/logs.go b/components/ledger/pkg/storage/ledgerstore/logs.go index 99ac12f015..9c0c2d8920 100644 --- a/components/ledger/pkg/storage/ledgerstore/logs.go +++ b/components/ledger/pkg/storage/ledgerstore/logs.go @@ -116,7 +116,7 @@ func (j RawMessage) Value() (driver.Value, error) { return string(j), nil } -func (s *Store) InsertLogs(ctx context.Context, activeLogs []*core.ActiveLog) error { +func (s *Store) InsertLogs(ctx context.Context, activeLogs ...*core.ActiveLog) error { txn, err := s.schema.BeginTx(ctx, &sql.TxOptions{}) if err != nil { diff --git a/components/ledger/pkg/storage/ledgerstore/logs_test.go b/components/ledger/pkg/storage/ledgerstore/logs_test.go index 9dc48605b4..55d550ab7c 100644 --- a/components/ledger/pkg/storage/ledgerstore/logs_test.go +++ b/components/ledger/pkg/storage/ledgerstore/logs_test.go @@ -359,7 +359,7 @@ func TestGetBalanceFromLogs(t *testing.T) { previousLog = activeLog.ChainedLog } } - err := store.InsertLogs(context.Background(), logs) + err := store.InsertLogs(context.Background(), logs...) require.NoError(t, err) balance, err := store.GetBalanceFromLogs(context.Background(), "account:1", "EUR/2") @@ -408,7 +408,7 @@ func TestGetMetadataFromLogs(t *testing.T) { map[string]metadata.Metadata{}, ).ChainLog(logs[3].ChainedLog))) - err := store.InsertLogs(context.Background(), logs) + err := store.InsertLogs(context.Background(), logs...) require.NoError(t, err) metadata, err := store.GetMetadataFromLogs(context.Background(), "bank", "foo") diff --git a/components/ledger/pkg/storage/ledgerstore/logs_worker.go b/components/ledger/pkg/storage/ledgerstore/logs_worker.go deleted file mode 100644 index 2f51f3ccd6..0000000000 --- a/components/ledger/pkg/storage/ledgerstore/logs_worker.go +++ /dev/null @@ -1,111 +0,0 @@ -package ledgerstore - -import ( - "context" - - "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/stack/libs/go-libs/collectionutils" - "github.com/formancehq/stack/libs/go-libs/logging" -) - -type pendingLog struct { - *core.LogPersistenceTracker - log *core.ActiveLog -} - -func (s *Store) AppendLog(ctx context.Context, log *core.ActiveLog) (*core.LogPersistenceTracker, error) { - ret := core.NewLogPersistenceTracker(log) - - select { - case <-ctx.Done(): - return nil, ctx.Err() - case s.writeChannel <- pendingLog{ - LogPersistenceTracker: ret, - log: log, - }: - return ret, nil - } -} - -func (s *Store) processPendingLogs(ctx context.Context, pendingLogs []pendingLog) { - models := make([]*core.ActiveLog, 0) - for _, holder := range pendingLogs { - models = append(models, holder.log) - } - err := s.InsertLogs(ctx, models) - if err != nil { - panic(err) - } - for _, holder := range pendingLogs { - holder.Resolve() - } - for _, f := range s.onLogsWrote { - f(collectionutils.Map(pendingLogs, func(from pendingLog) *core.ActiveLog { - return from.log - })) - } -} - -func (s *Store) Run(ctx context.Context) { - writeLoopStopped := make(chan struct{}) - effectiveSendChannel := make(chan []pendingLog) - stopped := make(chan struct{}) - go func() { - defer close(writeLoopStopped) - for { - select { - case <-stopped: - return - case pendingLogs := <-effectiveSendChannel: - s.processPendingLogs(ctx, pendingLogs) - } - } - }() - - var ( - sendChannel chan []pendingLog - bufferedPendingLogs = make([]pendingLog, 0) - logger = logging.FromContext(ctx) - ) - for { - select { - case ch := <-s.stopChan: - logger.Debugf("Terminating store worker, waiting end of write loop") - close(stopped) - <-writeLoopStopped - logger.Debugf("Write loop terminated, store properly closed") - //if len(bufferedPendingLogs) > 0 { - // s.processPendingLogs(ctx, bufferedPendingLogs) - //} - close(ch) - - return - case mh := <-s.writeChannel: - bufferedPendingLogs = append(bufferedPendingLogs, mh) - sendChannel = effectiveSendChannel - case sendChannel <- bufferedPendingLogs: - bufferedPendingLogs = make([]pendingLog, 0) - sendChannel = nil - } - } -} - -func (s *Store) Stop(ctx context.Context) error { - logging.FromContext(ctx).Info("Close store") - ch := make(chan struct{}) - select { - case <-ctx.Done(): - logging.FromContext(ctx).Errorf("Unable to close store: %s", ctx.Err()) - return ctx.Err() - case s.stopChan <- ch: - logging.FromContext(ctx).Debugf("Signal sent, waiting response") - select { - case <-ch: - logging.FromContext(ctx).Info("Store closed") - return nil - case <-ctx.Done(): - logging.FromContext(ctx).Errorf("Unable to close store: %s", ctx.Err()) - return ctx.Err() - } - } -} diff --git a/components/ledger/pkg/storage/ledgerstore/main_test.go b/components/ledger/pkg/storage/ledgerstore/main_test.go index da7af7453a..ac1c6e1f67 100644 --- a/components/ledger/pkg/storage/ledgerstore/main_test.go +++ b/components/ledger/pkg/storage/ledgerstore/main_test.go @@ -49,7 +49,7 @@ func newLedgerStore(t *testing.T) *ledgerstore.Store { require.NoError(t, db.Close()) }) - driver := driver.New("postgres", storage.NewDatabase(db), ledgerstore.DefaultStoreConfig) + driver := driver.New("postgres", storage.NewDatabase(db)) require.NoError(t, driver.Initialize(context.Background())) ledgerStore, err := driver.CreateLedgerStore(context.Background(), uuid.NewString()) require.NoError(t, err) @@ -61,10 +61,9 @@ func newLedgerStore(t *testing.T) *ledgerstore.Store { } func appendLog(t *testing.T, store *ledgerstore.Store, log *core.ChainedLog) *core.ChainedLog { - ret, err := store.AppendLog(context.Background(), core.NewActiveLog(log)) - <-ret.Done() + err := store.InsertLogs(context.Background(), core.NewActiveLog(log)) require.NoError(t, err) - return ret.ActiveLog().ChainedLog + return log } type explainHook struct{} diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go index 5421e76372..402f7aaaf7 100644 --- a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go +++ b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go @@ -263,7 +263,6 @@ func UpgradeLogs( func(ctx context.Context) error { return nil }, - ledgerstore.DefaultStoreConfig, ) if err != nil { return errors.Wrap(err, "creating store") diff --git a/components/ledger/pkg/storage/ledgerstore/store.go b/components/ledger/pkg/storage/ledgerstore/store.go index 46d26f6ee3..74e28e0f95 100644 --- a/components/ledger/pkg/storage/ledgerstore/store.go +++ b/components/ledger/pkg/storage/ledgerstore/store.go @@ -4,7 +4,6 @@ import ( "context" "sync" - "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/migrations" _ "github.com/jackc/pgx/v5/stdlib" @@ -15,45 +14,15 @@ const ( SQLCustomFuncMetaCompare = "meta_compare" ) -// TODO(gfyrag): useless, we have to throttle the application at higher level -type Config struct { - MaxPendingSize int - MaxWriteChanSize int -} - -var ( - DefaultConfig = Config{ - MaxPendingSize: 0, - MaxWriteChanSize: 1024, - } -) - -type OnLogWrote func([]*core.ActiveLog) - type Store struct { - schema storage.Schema - storeConfig StoreConfig - onDelete func(ctx context.Context) error + schema storage.Schema + onDelete func(ctx context.Context) error once sync.Once isInitialized bool - - writeChannel chan pendingLog - stopChan chan chan struct{} - onLogsWrote []OnLogWrote } -type StoreConfig struct { - StoreWorkerConfig Config -} - -var ( - DefaultStoreConfig = StoreConfig{ - StoreWorkerConfig: DefaultConfig, - } -) - func (s *Store) Schema() storage.Schema { return s.schema } @@ -87,20 +56,12 @@ func (s *Store) IsInitialized() bool { return s.isInitialized } -func (s *Store) OnLogWrote(fn OnLogWrote) { - s.onLogsWrote = append(s.onLogsWrote, fn) -} - func New( schema storage.Schema, onDelete func(ctx context.Context) error, - storeConfig StoreConfig, ) (*Store, error) { return &Store{ - schema: schema, - onDelete: onDelete, - storeConfig: storeConfig, - writeChannel: make(chan pendingLog, storeConfig.StoreWorkerConfig.MaxWriteChanSize), - stopChan: make(chan chan struct{}, 1), + schema: schema, + onDelete: onDelete, }, nil } diff --git a/components/ledger/pkg/storage/ledgerstore/transactions.go b/components/ledger/pkg/storage/ledgerstore/transactions.go index c548d3cc1a..5941547136 100644 --- a/components/ledger/pkg/storage/ledgerstore/transactions.go +++ b/components/ledger/pkg/storage/ledgerstore/transactions.go @@ -23,7 +23,6 @@ const ( MovesTableName = "moves" ) - type TransactionsQuery ColumnPaginatedQuery[TransactionsQueryFilters] func NewTransactionsQuery() TransactionsQuery { diff --git a/components/ledger/pkg/storage/storagetesting/storage.go b/components/ledger/pkg/storage/storagetesting/storage.go index 77c67fc7b0..60a790a236 100644 --- a/components/ledger/pkg/storage/storagetesting/storage.go +++ b/components/ledger/pkg/storage/storagetesting/storage.go @@ -7,7 +7,6 @@ import ( "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/driver" - "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/pgtesting" "github.com/stretchr/testify/require" ) @@ -28,7 +27,7 @@ func StorageDriver(t pgtesting.TestingT) *driver.Driver { db.Close() }) - d := driver.New("postgres", storage.NewDatabase(db), ledgerstore.DefaultStoreConfig) + d := driver.New("postgres", storage.NewDatabase(db)) require.NoError(t, d.Initialize(context.Background())) diff --git a/components/ledger/pkg/storage/tx.go b/components/ledger/pkg/storage/tx.go index f925c508dc..d217173228 100644 --- a/components/ledger/pkg/storage/tx.go +++ b/components/ledger/pkg/storage/tx.go @@ -20,4 +20,3 @@ func (s *Tx) NewInsert(tableName string) *bun.InsertQuery { func (s *Tx) NewUpdate(tableName string) *bun.UpdateQuery { return s.Tx.NewUpdate().ModelTableExpr("?0.?1 as ?1", bun.Ident(s.schema.Name()), bun.Ident(tableName)) } - From b089e52be8aa9faa241562e3ee2e196115261bd7 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 13 Jul 2023 17:46:18 +0200 Subject: [PATCH 3/7] fix: lock --- .../libs/collectionutils/linked_list.go | 16 +++- components/ledger/pkg/ledger/command/lock.go | 93 +++++++++---------- .../ledger/pkg/ledger/command/lock_test.go | 44 +++++++++ 3 files changed, 99 insertions(+), 54 deletions(-) create mode 100644 components/ledger/pkg/ledger/command/lock_test.go diff --git a/components/ledger/libs/collectionutils/linked_list.go b/components/ledger/libs/collectionutils/linked_list.go index b6ecb8ff97..c03a481b47 100644 --- a/components/ledger/libs/collectionutils/linked_list.go +++ b/components/ledger/libs/collectionutils/linked_list.go @@ -33,6 +33,14 @@ func (n *LinkedListNode[T]) Remove() { } } +func (n *LinkedListNode[T]) ForEach(f func(t T)) { + f(n.object) + if n.nextNode == nil { + return + } + n.nextNode.ForEach(f) +} + type LinkedList[T any] struct { mu sync.Mutex firstNode, lastNode *LinkedListNode[T] @@ -116,11 +124,11 @@ func (r *LinkedList[T]) ForEach(f func(t T)) { r.mu.Lock() defer r.mu.Unlock() - node := r.firstNode - for node != nil { - f(node.object) - node = node.nextNode + if r.firstNode == nil { + return } + + r.firstNode.ForEach(f) } func (r *LinkedList[T]) Slice() []T { diff --git a/components/ledger/pkg/ledger/command/lock.go b/components/ledger/pkg/ledger/command/lock.go index cd3f36b8ec..b5f7113acf 100644 --- a/components/ledger/pkg/ledger/command/lock.go +++ b/components/ledger/pkg/ledger/command/lock.go @@ -55,7 +55,7 @@ func (intent *lockIntent) tryLock(ctx context.Context, chain *DefaultLocker) boo } } - logging.FromContext(ctx).Debugf("Lock acquired, read: %s, write: %s", intent.accounts.Read, intent.accounts.Write) + logging.FromContext(ctx).Debugf("Lock acquired") for _, account := range intent.accounts.Read { atomicValue, ok := chain.readLocks[account] @@ -73,7 +73,7 @@ func (intent *lockIntent) tryLock(ctx context.Context, chain *DefaultLocker) boo } func (intent *lockIntent) unlock(ctx context.Context, chain *DefaultLocker) { - logging.FromContext(ctx).Debugf("Unlock accounts, read: %s, write: %s", intent.accounts.Read, intent.accounts.Write) + logging.FromContext(ctx).Debugf("Unlock accounts") for _, account := range intent.accounts.Read { atomicValue := chain.readLocks[account] if atomicValue.Add(-1) == 0 { @@ -94,67 +94,60 @@ type DefaultLocker struct { func (defaultLocker *DefaultLocker) Lock(ctx context.Context, accounts Accounts) (Unlock, error) { defaultLocker.mu.Lock() - defer defaultLocker.mu.Unlock() logger := logging.FromContext(ctx).WithFields(map[string]any{ "read": accounts.Read, "write": accounts.Write, }) + ctx = logging.ContextWithLogger(ctx, logger) logger.Debugf("Intent lock") intent := &lockIntent{ accounts: accounts, acquired: make(chan struct{}), } - if acquired := intent.tryLock(logging.ContextWithLogger(ctx, logger), defaultLocker); !acquired { - logger.Debugf("Lock not acquired, some accounts are already used") - - defaultLocker.intents.Append(intent) - select { - case <-ctx.Done(): - return nil, errors.Wrapf(ctx.Err(), "locking accounts: %s as read, and %s as write", accounts.Read, accounts.Write) - case <-intent.acquired: - return func(ctx context.Context) { - defaultLocker.mu.Lock() - defer defaultLocker.mu.Unlock() - - intent.unlock(ctx, defaultLocker) - node := defaultLocker.intents.RemoveValue(intent) - - if node == nil { - panic("node should not be nil") - } - - for { - node = node.Next() - if node == nil { - break - } - if node.Value().tryLock(ctx, defaultLocker) { - close(node.Value().acquired) - } - } - }, nil + + recheck := func() { + node := defaultLocker.intents.FirstNode() + for { + if node == nil { + break + } + if node.Value().tryLock(ctx, defaultLocker) { + node.Remove() + close(node.Value().acquired) + } + node = node.Next() } - } else { + } + + releaseIntent := func(ctx context.Context) { + defaultLocker.mu.Lock() + defer defaultLocker.mu.Unlock() + + intent.unlock(logging.ContextWithLogger(ctx, logger), defaultLocker) + + recheck() + } + + acquired := intent.tryLock(ctx, defaultLocker) + if acquired { + defaultLocker.mu.Unlock() logger.Debugf("Lock directly acquired") - return func(ctx context.Context) { - defaultLocker.mu.Lock() - defer defaultLocker.mu.Unlock() - - intent.unlock(ctx, defaultLocker) - - node := defaultLocker.intents.FirstNode() - for { - if node == nil { - break - } - if node.Value().tryLock(ctx, defaultLocker) { - close(node.Value().acquired) - } - node = node.Next() - } - }, nil + + return releaseIntent, nil + } + + logger.Debugf("Lock not acquired, some accounts are already used, putting in queue") + defaultLocker.intents.Append(intent) + defaultLocker.mu.Unlock() + + select { + case <-ctx.Done(): + defaultLocker.intents.RemoveValue(intent) + return nil, errors.Wrapf(ctx.Err(), "locking accounts: %s as read, and %s as write", accounts.Read, accounts.Write) + case <-intent.acquired: + return releaseIntent, nil } } diff --git a/components/ledger/pkg/ledger/command/lock_test.go b/components/ledger/pkg/ledger/command/lock_test.go new file mode 100644 index 0000000000..2e7fb1cfc5 --- /dev/null +++ b/components/ledger/pkg/ledger/command/lock_test.go @@ -0,0 +1,44 @@ +package command + +import ( + "fmt" + "math/rand" + "sync" + "testing" + "time" + + "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/stretchr/testify/require" +) + +func TestLock(t *testing.T) { + locker := NewDefaultLocker() + var accounts []string + for i := 0; i < 10; i++ { + accounts = append(accounts, fmt.Sprintf("accounts:%d", i)) + } + + r := rand.New(rand.NewSource(time.Now().Unix())) + ctx := logging.TestingContext() + + const nbLoop = 1000 + wg := sync.WaitGroup{} + wg.Add(nbLoop) + + for i := 0; i < nbLoop; i++ { + go func() { + unlock, err := locker.Lock(ctx, Accounts{ + Read: []string{accounts[r.Int31n(10)]}, + Write: []string{accounts[r.Int31n(10)]}, + }) + require.NoError(t, err) + defer unlock(ctx) + + <-time.After(10 * time.Millisecond) + wg.Add(-1) + }() + } + + wg.Wait() + +} From c47ae351c2b5693062e035bc04aa426c1e3aa4e1 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 13 Jul 2023 18:41:10 +0200 Subject: [PATCH 4/7] feat: refine database indices --- .../migrates/0-init-schema/postgres.sql | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql index 42183747f1..6cd0309dd6 100644 --- a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql +++ b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql @@ -18,32 +18,26 @@ create function "VAR_LEDGER_NAME_v2_0_0".meta_compare(metadata jsonb, value char --statement create table "VAR_LEDGER_NAME_v2_0_0".accounts ( - address character varying not null, + address character varying not null primary key, address_json jsonb not null, - metadata jsonb default '{}'::jsonb, - - unique(address) + metadata jsonb default '{}'::jsonb ); --statement create table "VAR_LEDGER_NAME_v2_0_0".logs_v2 ( - id bigint, + id numeric primary key , type smallint, hash bytea, date timestamp with time zone, data jsonb, idempotency_key varchar(255), - projected boolean default false, - - unique(id) + projected boolean default false ); --statement create table "VAR_LEDGER_NAME_v2_0_0".migrations_v2 ( - version character varying, - date character varying, - - unique(version) + version character varying primary key, + date character varying ); --statement From abaebf4a07dab8e2c934fdb42c7674979f9b94cf Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Fri, 14 Jul 2023 16:32:17 +0200 Subject: [PATCH 5/7] fix: migrations --- components/ledger/cmd/root.go | 1 + components/ledger/cmd/storage.go | 110 +++++++++++----- .../libs/collectionutils/linked_list.go | 16 +-- components/ledger/libs/service/logging.go | 8 +- .../ledger/pkg/ledger/query/projector.go | 25 +++- .../ledger/pkg/ledger/utils/job/jobs.go | 9 ++ components/ledger/pkg/storage/database.go | 4 - components/ledger/pkg/storage/driver/cli.go | 41 +++--- .../ledger/pkg/storage/driver/driver.go | 16 +-- components/ledger/pkg/storage/flags.go | 28 +++++ .../ledger/pkg/storage/ledgerstore/logs.go | 24 ++-- .../pkg/storage/ledgerstore/main_test.go | 2 +- .../ledgerstore/migrates/0-init-schema/any.go | 118 ++++++++++++------ .../migrates/0-init-schema/postgres.sql | 14 +-- .../pkg/storage/ledgerstore/transactions.go | 3 - .../pkg/storage/migrations/migrations.go | 14 ++- .../pkg/storage/storagetesting/storage.go | 2 +- libs/go-libs/service/logging.go | 8 +- 18 files changed, 289 insertions(+), 154 deletions(-) create mode 100644 components/ledger/pkg/storage/flags.go diff --git a/components/ledger/cmd/root.go b/components/ledger/cmd/root.go index e95bb4b63f..6d838b04d5 100644 --- a/components/ledger/cmd/root.go +++ b/components/ledger/cmd/root.go @@ -44,6 +44,7 @@ func NewRootCommand() *cobra.Command { store.AddCommand(NewStorageInit()) store.AddCommand(NewStorageList()) store.AddCommand(NewStorageUpgrade()) + store.AddCommand(NewStorageUpgradeAll()) store.AddCommand(NewStorageDelete()) root.AddCommand(serve) diff --git a/components/ledger/cmd/storage.go b/components/ledger/cmd/storage.go index 90186dfa0b..ce0aaad922 100644 --- a/components/ledger/cmd/storage.go +++ b/components/ledger/cmd/storage.go @@ -4,7 +4,9 @@ import ( "context" "errors" + "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/driver" + "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/stack/libs/go-libs/logging" "github.com/formancehq/stack/libs/go-libs/service" "github.com/spf13/cobra" @@ -95,37 +97,89 @@ func NewStorageList() *cobra.Command { return cmd } +func upgradeStore(ctx context.Context, store *ledgerstore.Store, name string) error { + modified, err := store.Migrate(ctx) + if err != nil { + return err + } + + if modified { + logging.FromContext(ctx).Infof("Storage '%s' upgraded", name) + } else { + logging.FromContext(ctx).Infof("Storage '%s' is up to date", name) + } + return nil +} + func NewStorageUpgrade() *cobra.Command { cmd := &cobra.Command{ - Use: "upgrade", - Args: cobra.ExactArgs(1), + Use: "upgrade", + Args: cobra.ExactArgs(1), + SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - app := service.New(cmd.OutOrStdout(), - resolveOptions( - cmd.OutOrStdout(), - fx.Invoke(func(storageDriver *driver.Driver, lc fx.Lifecycle) { - lc.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - name := args[0] - store, err := storageDriver.GetLedgerStore(ctx, name) - if err != nil { - return err - } - modified, err := store.Migrate(ctx) - if err != nil { - return err - } - if modified { - logging.FromContext(ctx).Infof("Storage '%s' upgraded", name) - } else { - logging.FromContext(ctx).Infof("Storage '%s' is up to date", name) - } - return nil - }, - }) - }))..., - ) - return app.Start(cmd.Context()) + + sqlDB, err := storage.OpenSQLDB(storage.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag))) + if err != nil { + return err + } + defer sqlDB.Close() + + driver := driver.New(storage.NewDatabase(sqlDB)) + if err := driver.Initialize(cmd.Context()); err != nil { + return err + } + + name := args[0] + store, err := driver.GetLedgerStore(cmd.Context(), name) + if err != nil { + return err + } + logger := service.GetDefaultLogger(cmd.OutOrStdout(), viper.GetBool(service.DebugFlag), false) + + return upgradeStore(logging.ContextWithLogger(cmd.Context(), logger), store, name) + }, + } + return cmd +} + +func NewStorageUpgradeAll() *cobra.Command { + cmd := &cobra.Command{ + Use: "upgrade-all", + Args: cobra.ExactArgs(0), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + + sqlDB, err := storage.OpenSQLDB(storage.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag))) + if err != nil { + return err + } + defer sqlDB.Close() + + driver := driver.New(storage.NewDatabase(sqlDB)) + if err := driver.Initialize(cmd.Context()); err != nil { + return err + } + logger := service.GetDefaultLogger(cmd.OutOrStdout(), viper.GetBool(service.DebugFlag), false) + ctx := logging.ContextWithLogger(cmd.Context(), logger) + + systemStore := driver.GetSystemStore() + ledgers, err := systemStore.ListLedgers(ctx) + if err != nil { + return err + } + + for _, ledger := range ledgers { + store, err := driver.GetLedgerStore(cmd.Context(), ledger) + if err != nil { + return err + } + logger.Infof("Upgrading storage '%s'", ledger) + if err := upgradeStore(ctx, store, ledger); err != nil { + return err + } + } + + return nil }, } return cmd diff --git a/components/ledger/libs/collectionutils/linked_list.go b/components/ledger/libs/collectionutils/linked_list.go index c03a481b47..b6ecb8ff97 100644 --- a/components/ledger/libs/collectionutils/linked_list.go +++ b/components/ledger/libs/collectionutils/linked_list.go @@ -33,14 +33,6 @@ func (n *LinkedListNode[T]) Remove() { } } -func (n *LinkedListNode[T]) ForEach(f func(t T)) { - f(n.object) - if n.nextNode == nil { - return - } - n.nextNode.ForEach(f) -} - type LinkedList[T any] struct { mu sync.Mutex firstNode, lastNode *LinkedListNode[T] @@ -124,11 +116,11 @@ func (r *LinkedList[T]) ForEach(f func(t T)) { r.mu.Lock() defer r.mu.Unlock() - if r.firstNode == nil { - return + node := r.firstNode + for node != nil { + f(node.object) + node = node.nextNode } - - r.firstNode.ForEach(f) } func (r *LinkedList[T]) Slice() []T { diff --git a/components/ledger/libs/service/logging.go b/components/ledger/libs/service/logging.go index 5d308c80b7..20821588f8 100644 --- a/components/ledger/libs/service/logging.go +++ b/components/ledger/libs/service/logging.go @@ -11,7 +11,7 @@ import ( "github.com/uptrace/opentelemetry-go-extra/otellogrus" ) -func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonFormattingLog bool) context.Context { +func GetDefaultLogger(w io.Writer, debug, jsonFormattingLog bool) logging.Logger { l := logrus.New() l.SetOutput(w) if debug { @@ -40,5 +40,9 @@ func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonForma logrus.WarnLevel, ))) } - return logging.ContextWithLogger(parent, logging.NewLogrus(l)) + return logging.NewLogrus(l) +} + +func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonFormattingLog bool) context.Context { + return logging.ContextWithLogger(parent, GetDefaultLogger(w, debug, jsonFormattingLog)) } diff --git a/components/ledger/pkg/ledger/query/projector.go b/components/ledger/pkg/ledger/query/projector.go index 197c506f59..85030033eb 100644 --- a/components/ledger/pkg/ledger/query/projector.go +++ b/components/ledger/pkg/ledger/query/projector.go @@ -194,6 +194,7 @@ func (p *Projector) processLogs(ctx context.Context, logs []*core.ActiveLog) { markLogAsProjected() p.monitor.CommittedTransactions(ctx, *payload.Transaction) }) + dispatchTransaction(l, log, *payload.Transaction) case core.SetMetadataLogPayload: switch payload.TargetType { @@ -253,7 +254,29 @@ func NewProjector( 512, ), accountMetadataWorker: batching.NewBatcher( - store.UpdateAccountsMetadata, + // TODO: UpdateAccountsMetadata insert metadata by batch + // but a batch can contains twice the same accounts + // we should create a dedicated component (as for moves) + // to aggregate metadata update by account + func(ctx context.Context, accounts ...core.Account) error { + ret := make(map[string]core.Account) + for _, account := range accounts { + _, ok := ret[account.Address] + if !ok { + ret[account.Address] = account + } + + updatedAccount := ret[account.Address] + updatedAccount.Metadata = updatedAccount.Metadata.Merge(account.Metadata) + } + + effectiveAccounts := make([]core.Account, 0) + for _, account := range ret { + effectiveAccounts = append(effectiveAccounts, account) + } + + return store.UpdateAccountsMetadata(ctx, effectiveAccounts...) + }, batching.NoOpOnBatchProcessed[core.Account](), 1, 512, diff --git a/components/ledger/pkg/ledger/utils/job/jobs.go b/components/ledger/pkg/ledger/utils/job/jobs.go index dcd599c9f9..5538d81c56 100644 --- a/components/ledger/pkg/ledger/utils/job/jobs.go +++ b/components/ledger/pkg/ledger/utils/job/jobs.go @@ -3,6 +3,7 @@ package job import ( "context" "fmt" + "runtime/debug" "sync/atomic" "github.com/alitto/pond" @@ -53,6 +54,14 @@ func (r *Runner[JOB]) Run(ctx context.Context) { logger := logging.FromContext(ctx) logger.Infof("Start worker") + defer func() { + if e := recover(); e != nil { + logger.Error(e) + debug.PrintStack() + panic(e) + } + }() + terminatedJobs := make(chan *JOB, r.nbWorkers) jobsErrors := make(chan error, r.nbWorkers) diff --git a/components/ledger/pkg/storage/database.go b/components/ledger/pkg/storage/database.go index c78d5851cd..cee4ce71ac 100644 --- a/components/ledger/pkg/storage/database.go +++ b/components/ledger/pkg/storage/database.go @@ -29,10 +29,6 @@ func (p *Database) Schema(name string) (Schema, error) { }, nil } -func (p *Database) Close(ctx context.Context) error { - return p.db.Close() -} - func NewDatabase(db *bun.DB) *Database { return &Database{ db: db, diff --git a/components/ledger/pkg/storage/driver/cli.go b/components/ledger/pkg/storage/driver/cli.go index f2d3e1d8f1..ca7f38990d 100644 --- a/components/ledger/pkg/storage/driver/cli.go +++ b/components/ledger/pkg/storage/driver/cli.go @@ -1,6 +1,7 @@ package driver import ( + "context" "io" "time" @@ -12,25 +13,16 @@ import ( "go.uber.org/fx" ) -const ( - StoreWorkerMaxPendingSize = "store-worker-max-pending-size" - StoreWorkerMaxWriteChanSize = "store-worker-max-write-chan-size" - StoragePostgresConnectionStringFlag = "storage-postgres-conn-string" - StoragePostgresMaxIdleConnsFlag = "storage-postgres-max-idle-conns" - StoragePostgresConnMaxIdleTimeFlag = "storage-postgres-conn-max-idle-time" - StoragePostgresMaxOpenConns = "storage-postgres-max-open-conns" -) - // TODO(gfyrag): maybe move flag handling inside cmd/internal (as telemetry flags) // Or make the inverse (move analytics flags to pkg/analytics) // IMO, flags are more easily discoverable if located inside cmd/ func InitCLIFlags(cmd *cobra.Command) { - cmd.PersistentFlags().Int(StoreWorkerMaxPendingSize, 0, "Max pending size for store worker") - cmd.PersistentFlags().Int(StoreWorkerMaxWriteChanSize, 1024, "Max write channel size for store worker") - cmd.PersistentFlags().String(StoragePostgresConnectionStringFlag, "postgresql://localhost/postgres", "Postgres connection string") - cmd.PersistentFlags().Int(StoragePostgresMaxIdleConnsFlag, 20, "Max idle connections to database") - cmd.PersistentFlags().Duration(StoragePostgresConnMaxIdleTimeFlag, time.Minute, "Max idle time of idle connections") - cmd.PersistentFlags().Int(StoragePostgresMaxOpenConns, 20, "Max open connections") + cmd.PersistentFlags().Int(storage.StoreWorkerMaxPendingSize, 0, "Max pending size for store worker") + cmd.PersistentFlags().Int(storage.StoreWorkerMaxWriteChanSize, 1024, "Max write channel size for store worker") + cmd.PersistentFlags().String(storage.StoragePostgresConnectionStringFlag, "postgresql://localhost/postgres", "Postgres connection string") + cmd.PersistentFlags().Int(storage.StoragePostgresMaxIdleConnsFlag, 20, "Max idle connections to database") + cmd.PersistentFlags().Duration(storage.StoragePostgresConnMaxIdleTimeFlag, time.Minute, "Max idle time of idle connections") + cmd.PersistentFlags().Int(storage.StoragePostgresMaxOpenConns, 20, "Max open connections") } type PostgresConfig struct { @@ -46,29 +38,26 @@ func CLIModule(v *viper.Viper, output io.Writer, debug bool) fx.Option { options := make([]fx.Option, 0) options = append(options, fx.Provide(func() (*bun.DB, error) { - return storage.OpenSQLDB(storage.ConnectionOptions{ - DatabaseSourceName: v.GetString(StoragePostgresConnectionStringFlag), - Debug: debug, - Writer: output, - MaxIdleConns: v.GetInt(StoragePostgresMaxIdleConnsFlag), - ConnMaxIdleTime: v.GetDuration(StoragePostgresConnMaxIdleTimeFlag), - MaxOpenConns: v.GetInt(StoragePostgresMaxOpenConns), - }) + return storage.OpenSQLDB(storage.ConnectionOptionsFromFlags(v, output, debug)) })) options = append(options, fx.Provide(func(db *bun.DB) *storage.Database { return storage.NewDatabase(db) })) options = append(options, fx.Provide(func(db *storage.Database) (*Driver, error) { - return New("postgres", db), nil + return New(db), nil })) options = append(options, health.ProvideHealthCheck(func(db *bun.DB) health.NamedCheck { return health.NewNamedCheck("postgres", health.CheckFn(db.PingContext)) })) - options = append(options, fx.Invoke(func(driver *Driver, lifecycle fx.Lifecycle) error { + options = append(options, fx.Invoke(func(db *bun.DB, driver *Driver, lifecycle fx.Lifecycle) error { lifecycle.Append(fx.Hook{ OnStart: driver.Initialize, - OnStop: driver.Close, + }) + lifecycle.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + return db.Close() + }, }) return nil })) diff --git a/components/ledger/pkg/storage/driver/driver.go b/components/ledger/pkg/storage/driver/driver.go index ba4ee7d9e4..0a415cf425 100644 --- a/components/ledger/pkg/storage/driver/driver.go +++ b/components/ledger/pkg/storage/driver/driver.go @@ -65,7 +65,6 @@ func InstrumentalizeSQLDriver() { } type Driver struct { - name string db *storage.Database systemStore *systemstore.Store lock sync.Mutex @@ -140,12 +139,8 @@ func (d *Driver) GetLedgerStore(ctx context.Context, name string) (*ledgerstore. return d.newStore(ctx, name) } -func (d *Driver) Name() string { - return d.name -} - func (d *Driver) Initialize(ctx context.Context) error { - logging.FromContext(ctx).Debugf("Initialize driver %s", d.name) + logging.FromContext(ctx).Debugf("Initialize driver") if err := d.db.Initialize(ctx); err != nil { return err @@ -169,13 +164,8 @@ func (d *Driver) Initialize(ctx context.Context) error { return nil } -func (d *Driver) Close(ctx context.Context) error { - return d.db.Close(ctx) -} - -func New(name string, db *storage.Database) *Driver { +func New(db *storage.Database) *Driver { return &Driver{ - db: db, - name: name, + db: db, } } diff --git a/components/ledger/pkg/storage/flags.go b/components/ledger/pkg/storage/flags.go new file mode 100644 index 0000000000..351dfb9228 --- /dev/null +++ b/components/ledger/pkg/storage/flags.go @@ -0,0 +1,28 @@ +package storage + +import ( + "io" + + "github.com/spf13/viper" +) + +const ( + StoreWorkerMaxPendingSize = "store-worker-max-pending-size" + StoreWorkerMaxWriteChanSize = "store-worker-max-write-chan-size" + StoragePostgresConnectionStringFlag = "storage-postgres-conn-string" + StoragePostgresMaxIdleConnsFlag = "storage-postgres-max-idle-conns" + StoragePostgresConnMaxIdleTimeFlag = "storage-postgres-conn-max-idle-time" + StoragePostgresMaxOpenConns = "storage-postgres-max-open-conns" +) + +func ConnectionOptionsFromFlags(v *viper.Viper, output io.Writer, debug bool) ConnectionOptions { + return ConnectionOptions{ + DatabaseSourceName: v.GetString(StoragePostgresConnectionStringFlag), + Debug: debug, + Trace: debug, + Writer: output, + MaxIdleConns: v.GetInt(StoragePostgresMaxIdleConnsFlag), + ConnMaxIdleTime: v.GetDuration(StoragePostgresConnMaxIdleTimeFlag), + MaxOpenConns: v.GetInt(StoragePostgresMaxOpenConns), + } +} diff --git a/components/ledger/pkg/storage/ledgerstore/logs.go b/components/ledger/pkg/storage/ledgerstore/logs.go index 9c0c2d8920..75744efdeb 100644 --- a/components/ledger/pkg/storage/ledgerstore/logs.go +++ b/components/ledger/pkg/storage/ledgerstore/logs.go @@ -20,7 +20,7 @@ import ( ) const ( - LogTableName = "logs_v2" + LogTableName = "logs" ) type LogsQueryFilters struct { @@ -77,7 +77,7 @@ type AccountWithBalances struct { } type LogsV2 struct { - bun.BaseModel `bun:"logs_v2,alias:logs_v2"` + bun.BaseModel `bun:"logs,alias:logs"` ID uint64 `bun:"id,unique,type:bigint"` Type int16 `bun:"type,type:smallint"` @@ -88,7 +88,7 @@ type LogsV2 struct { Projected bool `bun:"projected,type:boolean"` } -func (log LogsV2) toCore() core.ChainedLog { +func (log LogsV2) ToCore() core.ChainedLog { payload, err := core.HydrateLog(core.LogType(log.Type), log.Data) if err != nil { panic(errors.Wrap(err, "hydrating log data")) @@ -126,7 +126,7 @@ func (s *Store) InsertLogs(ctx context.Context, activeLogs ...*core.ActiveLog) e // Beware: COPY query is not supported by bun if the pgx driver is used. stmt, err := txn.Prepare(pq.CopyInSchema( s.schema.Name(), - "logs_v2", + LogTableName, "id", "type", "hash", "date", "data", "idempotency_key", )) if err != nil { @@ -179,7 +179,7 @@ func (s *Store) GetLastLog(ctx context.Context) (*core.ChainedLog, error) { return nil, storageerrors.PostgresError(err) } - l := raw.toCore() + l := raw.ToCore() return &l, nil } @@ -192,7 +192,7 @@ func (s *Store) GetLogs(ctx context.Context, q LogsQuery) (*api.Cursor[core.Chai return nil, err } - return api.MapCursor(cursor, LogsV2.toCore), nil + return api.MapCursor(cursor, LogsV2.ToCore), nil } func (s *Store) buildLogsQuery(q LogsQueryFilters, models *[]LogsV2) *bun.SelectQuery { @@ -236,7 +236,7 @@ func (s *Store) ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core. return nil, storageerrors.PostgresError(err) } - return collectionutils.Map(rawLogs, LogsV2.toCore), nil + return collectionutils.Map(rawLogs, LogsV2.ToCore), nil } func (s *Store) ReadLastLogWithType(ctx context.Context, logTypes ...core.LogType) (*core.ChainedLog, error) { @@ -251,7 +251,7 @@ func (s *Store) ReadLastLogWithType(ctx context.Context, logTypes ...core.LogTyp if err != nil { return nil, storageerrors.PostgresError(err) } - ret := raw.toCore() + ret := raw.ToCore() return &ret, nil } @@ -268,7 +268,7 @@ func (s *Store) ReadLogForCreatedTransactionWithReference(ctx context.Context, r return nil, storageerrors.PostgresError(err) } - l := raw.toCore() + l := raw.ToCore() return &l, nil } @@ -284,7 +284,7 @@ func (s *Store) ReadLogForCreatedTransaction(ctx context.Context, txID uint64) ( return nil, storageerrors.PostgresError(err) } - l := raw.toCore() + l := raw.ToCore() return &l, nil } @@ -300,7 +300,7 @@ func (s *Store) ReadLogForRevertedTransaction(ctx context.Context, txID uint64) return nil, storageerrors.PostgresError(err) } - l := raw.toCore() + l := raw.ToCore() return &l, nil } @@ -316,7 +316,7 @@ func (s *Store) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*cor return nil, storageerrors.PostgresError(err) } - l := raw.toCore() + l := raw.ToCore() return &l, nil } diff --git a/components/ledger/pkg/storage/ledgerstore/main_test.go b/components/ledger/pkg/storage/ledgerstore/main_test.go index ac1c6e1f67..23a1dc04bd 100644 --- a/components/ledger/pkg/storage/ledgerstore/main_test.go +++ b/components/ledger/pkg/storage/ledgerstore/main_test.go @@ -49,7 +49,7 @@ func newLedgerStore(t *testing.T) *ledgerstore.Store { require.NoError(t, db.Close()) }) - driver := driver.New("postgres", storage.NewDatabase(db)) + driver := driver.New(storage.NewDatabase(db)) require.NoError(t, driver.Initialize(context.Background())) ledgerStore, err := driver.CreateLedgerStore(context.Background(), uuid.NewString()) require.NoError(t, err) diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go index 402f7aaaf7..75b9caa4de 100644 --- a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go +++ b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go @@ -7,8 +7,6 @@ import ( "fmt" "github.com/formancehq/ledger/pkg/core" - "github.com/formancehq/ledger/pkg/ledger/query" - "github.com/formancehq/ledger/pkg/opentelemetry/metrics" "github.com/formancehq/ledger/pkg/storage" "github.com/formancehq/ledger/pkg/storage/ledgerstore" "github.com/formancehq/ledger/pkg/storage/migrations" @@ -78,44 +76,35 @@ func isLogTableExisting( func readLogsRange( ctx context.Context, - schema storage.Schema, sqlTx *storage.Tx, idMin, idMax uint64, ) ([]Log, error) { rawLogs := make([]Log, 0) - sb := schema. + if err := sqlTx. NewSelect(LogTableName). Where("id >= ?", idMin). Where("id < ?", idMax). - Model((*Log)(nil)) - - rows, err := sqlTx.QueryContext(ctx, sb.String()) - if err != nil { - if err == sql.ErrNoRows { - return rawLogs, nil - } - - return nil, errors.Wrap(err, "selecting logs") + Model(&rawLogs). + Scan(ctx); err != nil { + return nil, err } - defer func() { - if err := rows.Close(); err != nil { - if err == sql.ErrNoRows { - return - } - panic(err) - } - }() - for rows.Next() { - var log Log - if err := rows.Scan(&log); err != nil { - return nil, errors.Wrap(err, "scanning log") - } + return rawLogs, nil +} - rawLogs = append(rawLogs, log) +func convertMetadata(data []byte) any { + ret := make(map[string]any) + if err := json.Unmarshal(data, &ret); err != nil { + panic(err) + } + oldMetadata := ret["metadata"].(map[string]any) + newMetadata := make(map[string]string) + for k, v := range oldMetadata { + newMetadata[k] = fmt.Sprint(v) } + ret["metadata"] = newMetadata - return rawLogs, nil + return ret } func (l *Log) ToLogsV2() (ledgerstore.LogsV2, error) { @@ -124,12 +113,32 @@ func (l *Log) ToLogsV2() (ledgerstore.LogsV2, error) { return ledgerstore.LogsV2{}, errors.Wrap(err, "converting log type") } + var data any + switch logType { + case core.NewTransactionLogType: + data = map[string]any{ + "transaction": convertMetadata(l.Data), + "accountMetadata": map[string]any{}, + } + case core.SetMetadataLogType: + data = convertMetadata(l.Data) + case core.RevertedTransactionLogType: + data = l.Data + default: + panic("unknown type " + logType.String()) + } + + asJson, err := json.Marshal(data) + if err != nil { + panic(err) + } + return ledgerstore.LogsV2{ ID: l.ID, Type: int16(logType), Hash: []byte(l.Hash), Date: l.Date, - Data: l.Data, + Data: asJson, }, nil } @@ -147,7 +156,7 @@ func batchLogs( // Beware: COPY query is not supported by bun if the pgx driver is used. stmt, err := txn.Prepare(pq.CopyInSchema( schema.Name(), - "logs_v2", + ledgerstore.LogTableName, "id", "type", "hash", "date", "data", )) if err != nil { @@ -196,6 +205,7 @@ func migrateLogs( ctx context.Context, schemaV1 storage.Schema, schemaV2 storage.Schema, + store *ledgerstore.Store, sqlTx *storage.Tx, ) error { exists, err := isLogTableExisting(ctx, schemaV1, sqlTx) @@ -210,7 +220,7 @@ func migrateLogs( var idMin uint64 var idMax = idMin + batchSize for { - logs, err := readLogsRange(ctx, schemaV1, sqlTx, idMin, idMax) + logs, err := readLogsRange(ctx, sqlTx, idMin, idMax) if err != nil { return err } @@ -234,6 +244,46 @@ func migrateLogs( return err } + for _, log := range logsV2 { + coreLog := log.ToCore() + switch payload := coreLog.Data.(type) { + case core.NewTransactionLogPayload: + if err := store.InsertTransactions(ctx, *payload.Transaction); err != nil { + return err + } + case core.SetMetadataLogPayload: + switch payload.TargetType { + case core.MetaTargetTypeTransaction: + if err := store.UpdateTransactionsMetadata(ctx, core.TransactionWithMetadata{ + ID: payload.TargetID.(uint64), + Metadata: payload.Metadata, + }); err != nil { + return err + } + case core.MetaTargetTypeAccount: + if err := store.UpdateAccountsMetadata(ctx, core.Account{ + Address: payload.TargetID.(string), + Metadata: payload.Metadata, + }); err != nil { + return err + } + } + case core.RevertedTransactionLogPayload: + if err := store.InsertTransactions(ctx, *payload.RevertTransaction); err != nil { + return err + } + if err := store.UpdateTransactionsMetadata(ctx, core.TransactionWithMetadata{ + ID: payload.RevertedTransactionID, + Metadata: core.RevertedMetadata(payload.RevertTransaction.ID), + }); err != nil { + return err + } + } + if err := store.MarkedLogsAsProjected(ctx, log.ID); err != nil { + return err + } + } + idMin = idMax idMax = idMin + batchSize } @@ -268,13 +318,9 @@ func UpgradeLogs( return errors.Wrap(err, "creating store") } - if err := migrateLogs(ctx, schemaV1, schemaV2, sqlTx); err != nil { + if err := migrateLogs(ctx, schemaV1, schemaV2, store, sqlTx); err != nil { return errors.Wrap(err, "migrating logs") } - projector := query.NewProjector(store, query.NewNoOpMonitor(), metrics.NewNoOpRegistry()) - projector.Start(ctx) // Start block until logs are synced - projector.Stop(ctx) - return cleanSchema(ctx, schemaV1, schemaV2, sqlTx) } diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql index 6cd0309dd6..dc67fd1af2 100644 --- a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql +++ b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/postgres.sql @@ -24,7 +24,7 @@ create table "VAR_LEDGER_NAME_v2_0_0".accounts ( ); --statement -create table "VAR_LEDGER_NAME_v2_0_0".logs_v2 ( +create table "VAR_LEDGER_NAME_v2_0_0".logs ( id numeric primary key , type smallint, hash bytea, @@ -35,7 +35,7 @@ create table "VAR_LEDGER_NAME_v2_0_0".logs_v2 ( ); --statement -create table "VAR_LEDGER_NAME_v2_0_0".migrations_v2 ( +create table "VAR_LEDGER_NAME_v2_0_0".migrations ( version character varying primary key, date character varying ); @@ -65,19 +65,19 @@ create table "VAR_LEDGER_NAME_v2_0_0".moves ( ); --statement -create index logsv2_type on "VAR_LEDGER_NAME_v2_0_0".logs_v2 (type); +create index logsv2_type on "VAR_LEDGER_NAME_v2_0_0".logs (type); --statement -create index logsv2_projected on "VAR_LEDGER_NAME_v2_0_0".logs_v2 (projected); +create index logsv2_projected on "VAR_LEDGER_NAME_v2_0_0".logs (projected); --statement -create index logsv2_data on "VAR_LEDGER_NAME_v2_0_0".logs_v2 using gin (data); +create index logsv2_data on "VAR_LEDGER_NAME_v2_0_0".logs using gin (data); --statement -create index logsv2_new_transaction_postings on "VAR_LEDGER_NAME_v2_0_0".logs_v2 using gin ((data->'transaction'->'postings') jsonb_path_ops); +create index logsv2_new_transaction_postings on "VAR_LEDGER_NAME_v2_0_0".logs using gin ((data->'transaction'->'postings') jsonb_path_ops); --statement -create index logsv2_set_metadata on "VAR_LEDGER_NAME_v2_0_0".logs_v2 using btree (type, (data->>'targetId'), (data->>'targetType')); +create index logsv2_set_metadata on "VAR_LEDGER_NAME_v2_0_0".logs using btree (type, (data->>'targetId'), (data->>'targetType')); --statement create index transactions_id_timestamp on "VAR_LEDGER_NAME_v2_0_0".transactions(id, timestamp); diff --git a/components/ledger/pkg/storage/ledgerstore/transactions.go b/components/ledger/pkg/storage/ledgerstore/transactions.go index 5941547136..8e6f6591a4 100644 --- a/components/ledger/pkg/storage/ledgerstore/transactions.go +++ b/components/ledger/pkg/storage/ledgerstore/transactions.go @@ -453,9 +453,6 @@ func (s *Store) InsertMoves(ctx context.Context, objects ...*core.Move) error { if err != nil { return err } - defer func() { - _ = tx.Rollback() - }() err = tx.NewInsert(MovesTableName). With("cte1", s.schema.NewValues(&moves)). diff --git a/components/ledger/pkg/storage/migrations/migrations.go b/components/ledger/pkg/storage/migrations/migrations.go index 0ca83fa894..039a1bec0c 100644 --- a/components/ledger/pkg/storage/migrations/migrations.go +++ b/components/ledger/pkg/storage/migrations/migrations.go @@ -21,10 +21,10 @@ import ( "github.com/uptrace/bun" ) -const migrationsTableName = "migrations_v2" +const migrationsTableName = "migrations" type table struct { - bun.BaseModel `bun:"migrations_v2,alias:migrations_v2"` + bun.BaseModel `bun:"migrations,alias:migrations"` Version string `bun:"version,type:varchar,unique"` Date string `bun:"date,type:varchar"` @@ -55,8 +55,10 @@ func Migrate(ctx context.Context, s storage.Schema, migrations ...Migration) (bo _ = tx.Rollback() }(tx) + logging.FromContext(ctx).Debugf("Checking migrations...") modified := false for _, m := range migrations { + logging.FromContext(ctx).Debugf("Checking if version %s is applied", m.Version) sb := s.NewSelect(migrationsTableName). Model((*table)(nil)). Column("version"). @@ -66,15 +68,15 @@ func Migrate(ctx context.Context, s storage.Schema, migrations ...Migration) (bo row := s.QueryRowContext(ctx, sb.String()) var v string if err = row.Scan(&v); err != nil { - logging.FromContext(ctx).Debugf("migration %s: %s", m.Version, err) + logging.FromContext(ctx).Debugf("Migration %s: %s", m.Version, err) } if v != "" { - logging.FromContext(ctx).Debugf("migration %s: already up to date", m.Version) + logging.FromContext(ctx).Debugf("Migration %s: already up to date", m.Version) continue } modified = true - logging.FromContext(ctx).Debugf("running migration %s", m.Version) + logging.FromContext(ctx).Debugf("Running migration %s", m.Version) handlersForCurrentEngine, ok := m.Handlers["postgres"] if ok { @@ -103,7 +105,7 @@ func Migrate(ctx context.Context, s storage.Schema, migrations ...Migration) (bo sbInsert := s.NewInsert(migrationsTableName).Model(&m) if _, err := tx.ExecContext(ctx, sbInsert.String()); err != nil { - logging.FromContext(ctx).Errorf("failed to insert migration version %s: %s", m.Version, err) + logging.FromContext(ctx).Errorf("Failed to insert migration version %s: %s", m.Version, err) return false, storage.PostgresError(err) } diff --git a/components/ledger/pkg/storage/storagetesting/storage.go b/components/ledger/pkg/storage/storagetesting/storage.go index 60a790a236..c3eaac89b0 100644 --- a/components/ledger/pkg/storage/storagetesting/storage.go +++ b/components/ledger/pkg/storage/storagetesting/storage.go @@ -27,7 +27,7 @@ func StorageDriver(t pgtesting.TestingT) *driver.Driver { db.Close() }) - d := driver.New("postgres", storage.NewDatabase(db)) + d := driver.New(storage.NewDatabase(db)) require.NoError(t, d.Initialize(context.Background())) diff --git a/libs/go-libs/service/logging.go b/libs/go-libs/service/logging.go index 5d308c80b7..20821588f8 100644 --- a/libs/go-libs/service/logging.go +++ b/libs/go-libs/service/logging.go @@ -11,7 +11,7 @@ import ( "github.com/uptrace/opentelemetry-go-extra/otellogrus" ) -func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonFormattingLog bool) context.Context { +func GetDefaultLogger(w io.Writer, debug, jsonFormattingLog bool) logging.Logger { l := logrus.New() l.SetOutput(w) if debug { @@ -40,5 +40,9 @@ func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonForma logrus.WarnLevel, ))) } - return logging.ContextWithLogger(parent, logging.NewLogrus(l)) + return logging.NewLogrus(l) +} + +func defaultLoggingContext(parent context.Context, w io.Writer, debug, jsonFormattingLog bool) context.Context { + return logging.ContextWithLogger(parent, GetDefaultLogger(w, debug, jsonFormattingLog)) } From 09b81f6f70939961c1577073976843f43788ed95 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Fri, 14 Jul 2023 20:28:27 +0200 Subject: [PATCH 6/7] fix: metadata update conflict --- .../pkg/ledger/query/metadata_updater.go | 139 ++++++++++++++++++ .../query/{move_buffer.go => move_updater.go} | 2 +- ...ve_buffer_test.go => move_updater_test.go} | 2 +- .../ledger/pkg/ledger/query/projector.go | 99 +++++-------- components/ledger/pkg/storage/inmemory.go | 27 ++++ 5 files changed, 202 insertions(+), 67 deletions(-) create mode 100644 components/ledger/pkg/ledger/query/metadata_updater.go rename components/ledger/pkg/ledger/query/{move_buffer.go => move_updater.go} (95%) rename components/ledger/pkg/ledger/query/{move_buffer_test.go => move_updater_test.go} (93%) diff --git a/components/ledger/pkg/ledger/query/metadata_updater.go b/components/ledger/pkg/ledger/query/metadata_updater.go new file mode 100644 index 0000000000..9c2c8c5a6f --- /dev/null +++ b/components/ledger/pkg/ledger/query/metadata_updater.go @@ -0,0 +1,139 @@ +package query + +import ( + "context" + "fmt" + "sync" + + "github.com/formancehq/ledger/pkg/ledger/utils/job" + "github.com/formancehq/stack/libs/go-libs/collectionutils" + "github.com/formancehq/stack/libs/go-libs/metadata" +) + +type metadataUpdaterInput struct { + id any + metadata.Metadata + callback func() +} + +type inputs []*metadataUpdaterInput + +func (inputs inputs) aggregated() metadata.Metadata { + m := metadata.Metadata{} + for _, object := range inputs { + m = m.Merge(object.Metadata) + } + return m +} + +type metadataUpdaterBuffer struct { + id any + inputs inputs +} + +type metadataUpdaterJob struct { + updater *metadataUpdater + buffers []*metadataUpdaterBuffer + inputs []*metadataUpdaterInput + aggregated []*MetadataUpdate +} + +func (j metadataUpdaterJob) String() string { + return fmt.Sprintf("inserting %d objects", len(j.inputs)) +} + +func (j metadataUpdaterJob) Terminated() { + for _, input := range j.inputs { + input.callback() + } + + j.updater.mu.Lock() + defer j.updater.mu.Unlock() + + for _, buffer := range j.buffers { + if len(buffer.inputs) == 0 { + delete(j.updater.objects, buffer.id) + } else { + j.updater.queue.Append(buffer) + } + } +} + +type metadataUpdater struct { + *job.Runner[metadataUpdaterJob] + queue *collectionutils.LinkedList[*metadataUpdaterBuffer] + objects map[any]*metadataUpdaterBuffer + input chan *moveBufferInput + mu sync.Mutex + maxBufferSize int +} + +func (r *metadataUpdater) Append(id any, metadata metadata.Metadata, callback func()) { + r.mu.Lock() + + mba, ok := r.objects[id] + if !ok { + mba = &metadataUpdaterBuffer{ + id: id, + } + r.objects[id] = mba + r.queue.Append(mba) + } + mba.inputs = append(mba.inputs, &metadataUpdaterInput{ + id: id, + Metadata: metadata, + callback: callback, + }) + r.mu.Unlock() + + r.Runner.Next() +} + +func (r *metadataUpdater) nextJob() *metadataUpdaterJob { + r.mu.Lock() + defer r.mu.Unlock() + + batch := make([]*metadataUpdaterInput, 0) + aggregated := make([]*MetadataUpdate, 0) + for len(batch) < r.maxBufferSize { + mba := r.queue.TakeFirst() + if mba == nil { + break + } + + batch = append(batch, mba.inputs...) + aggregated = append(aggregated, &MetadataUpdate{ + ID: mba.id, + Metadata: mba.inputs.aggregated(), + }) + mba.inputs = inputs{} + } + + if len(batch) == 0 { + return nil + } + + return &metadataUpdaterJob{ + inputs: batch, + updater: r, + aggregated: aggregated, + } +} + +type MetadataUpdate struct { + ID any + Metadata metadata.Metadata +} + +func newMetadataUpdater(runner func(context.Context, ...*MetadataUpdate) error, nbWorkers, maxBufferSize int) *metadataUpdater { + ret := &metadataUpdater{ + queue: collectionutils.NewLinkedList[*metadataUpdaterBuffer](), + objects: map[any]*metadataUpdaterBuffer{}, + input: make(chan *moveBufferInput), + maxBufferSize: maxBufferSize, + } + ret.Runner = job.NewJobRunner[metadataUpdaterJob](func(ctx context.Context, job *metadataUpdaterJob) error { + return runner(ctx, job.aggregated...) + }, ret.nextJob, nbWorkers) + return ret +} diff --git a/components/ledger/pkg/ledger/query/move_buffer.go b/components/ledger/pkg/ledger/query/move_updater.go similarity index 95% rename from components/ledger/pkg/ledger/query/move_buffer.go rename to components/ledger/pkg/ledger/query/move_updater.go index 175018783f..d6ee7c699a 100644 --- a/components/ledger/pkg/ledger/query/move_buffer.go +++ b/components/ledger/pkg/ledger/query/move_updater.go @@ -109,7 +109,7 @@ func (r *moveBuffer) nextJob() *insertMovesJob { } } -func newMoveBuffer(runner func(context.Context, ...*core.Move) error, nbWorkers, maxBufferSize int) *moveBuffer { +func newMoveUpdater(runner func(context.Context, ...*core.Move) error, nbWorkers, maxBufferSize int) *moveBuffer { ret := &moveBuffer{ accountsQueue: collectionutils.NewLinkedList[*moveBufferAccount](), accounts: map[string]*moveBufferAccount{}, diff --git a/components/ledger/pkg/ledger/query/move_buffer_test.go b/components/ledger/pkg/ledger/query/move_updater_test.go similarity index 93% rename from components/ledger/pkg/ledger/query/move_buffer_test.go rename to components/ledger/pkg/ledger/query/move_updater_test.go index a295ee4f1b..9c920e066c 100644 --- a/components/ledger/pkg/ledger/query/move_buffer_test.go +++ b/components/ledger/pkg/ledger/query/move_updater_test.go @@ -15,7 +15,7 @@ func TestMoveBuffer(t *testing.T) { t.Parallel() locked := sync.Map{} - buf := newMoveBuffer(func(ctx context.Context, moves ...*core.Move) error { + buf := newMoveUpdater(func(ctx context.Context, moves ...*core.Move) error { accounts := make(map[string]struct{}) for _, move := range moves { accounts[move.Account] = struct{}{} diff --git a/components/ledger/pkg/ledger/query/projector.go b/components/ledger/pkg/ledger/query/projector.go index 85030033eb..cea359981b 100644 --- a/components/ledger/pkg/ledger/query/projector.go +++ b/components/ledger/pkg/ledger/query/projector.go @@ -53,11 +53,11 @@ type Projector struct { stopChan chan chan struct{} activeLogs *collectionutils.LinkedList[*core.ActiveLog] - txWorker *batching.Batcher[core.Transaction] - txMetadataWorker *batching.Batcher[core.TransactionWithMetadata] - accountMetadataWorker *batching.Batcher[core.Account] + txUpdater *batching.Batcher[core.Transaction] + txMetadataUpdater *metadataUpdater + accountMetadataUpdater *metadataUpdater - moveBuffer *moveBuffer + moveUpdater *moveBuffer limitReadLogs int } @@ -79,10 +79,10 @@ func (p *Projector) Start(ctx context.Context) { ctx = logging.ContextWithLogger(ctx, logger) - go p.moveBuffer.Run(logging.ContextWithField(ctx, "component", "moves buffer")) - go p.txWorker.Run(logging.ContextWithField(ctx, "component", "transactions buffer")) - go p.accountMetadataWorker.Run(logging.ContextWithField(ctx, "component", "accounts metadata buffer")) - go p.txMetadataWorker.Run(logging.ContextWithField(ctx, "component", "transactions metadata buffer")) + go p.moveUpdater.Run(logging.ContextWithField(ctx, "component", "moves buffer")) + go p.txUpdater.Run(logging.ContextWithField(ctx, "component", "transactions buffer")) + go p.accountMetadataUpdater.Run(logging.ContextWithField(ctx, "component", "accounts metadata buffer")) + go p.txMetadataUpdater.Run(logging.ContextWithField(ctx, "component", "transactions metadata buffer")) p.syncLogs(ctx) @@ -92,16 +92,16 @@ func (p *Projector) Start(ctx context.Context) { select { case ch := <-p.stopChan: logger.Debugf("Close move buffer") - p.moveBuffer.Close() + p.moveUpdater.Close() logger.Debugf("Stop transaction worker") - p.txWorker.Close() + p.txUpdater.Close() logger.Debugf("Stop account metadata worker") - p.accountMetadataWorker.Close() + p.accountMetadataUpdater.Close() logger.Debugf("Stop transaction metadata worker") - p.txMetadataWorker.Close() + p.txMetadataUpdater.Close() close(ch) return @@ -171,14 +171,14 @@ func (p *Projector) processLogs(ctx context.Context, logs []*core.ActiveLog) { l.Store(moveKey(move)) } - p.txWorker.Append(tx, func() { + p.txUpdater.Append(tx, func() { logger.Debugf("Transaction projected") l.Delete("tx") }) for _, move := range moves { move := move - p.moveBuffer.AppendMove(move, func() { + p.moveUpdater.AppendMove(move, func() { logger.WithFields(map[string]any{ "asset": move.Asset, "is_source": move.IsSource, @@ -199,18 +199,12 @@ func (p *Projector) processLogs(ctx context.Context, logs []*core.ActiveLog) { case core.SetMetadataLogPayload: switch payload.TargetType { case core.MetaTargetTypeAccount: - p.accountMetadataWorker.Append(core.Account{ - Address: payload.TargetID.(string), - Metadata: payload.Metadata, - }, func() { + p.accountMetadataUpdater.Append(payload.TargetID, payload.Metadata, func() { markLogAsProjected() p.monitor.SavedMetadata(ctx, payload.TargetType, fmt.Sprint(payload.TargetID), payload.Metadata) }) case core.MetaTargetTypeTransaction: - p.txMetadataWorker.Append(core.TransactionWithMetadata{ - ID: payload.TargetID.(uint64), - Metadata: payload.Metadata, - }, func() { + p.txMetadataUpdater.Append(payload.TargetID, payload.Metadata, func() { markLogAsProjected() p.monitor.SavedMetadata(ctx, payload.TargetType, fmt.Sprint(payload.TargetID), payload.Metadata) }) @@ -228,66 +222,41 @@ func (p *Projector) processLogs(ctx context.Context, logs []*core.ActiveLog) { }) l.Store("metadata") dispatchTransaction(l, log, *payload.RevertTransaction) - p.txMetadataWorker.Append(core.TransactionWithMetadata{ - ID: payload.RevertedTransactionID, - Metadata: core.RevertedMetadata(payload.RevertTransaction.ID), - }, func() { + p.txMetadataUpdater.Append(payload.RevertedTransactionID, core.RevertedMetadata(payload.RevertTransaction.ID), func() { l.Delete("metadata") }) } } } -func NewProjector( - store Store, - monitor Monitor, - metricsRegistry metrics.PerLedgerRegistry, -) *Projector { +func NewProjector(store Store, monitor Monitor, metricsRegistry metrics.PerLedgerRegistry) *Projector { return &Projector{ store: store, monitor: monitor, metricsRegistry: metricsRegistry, - txWorker: batching.NewBatcher( + txUpdater: batching.NewBatcher( store.InsertTransactions, batching.NoOpOnBatchProcessed[core.Transaction](), 2, 512, ), - accountMetadataWorker: batching.NewBatcher( - // TODO: UpdateAccountsMetadata insert metadata by batch - // but a batch can contains twice the same accounts - // we should create a dedicated component (as for moves) - // to aggregate metadata update by account - func(ctx context.Context, accounts ...core.Account) error { - ret := make(map[string]core.Account) - for _, account := range accounts { - _, ok := ret[account.Address] - if !ok { - ret[account.Address] = account - } - - updatedAccount := ret[account.Address] - updatedAccount.Metadata = updatedAccount.Metadata.Merge(account.Metadata) + accountMetadataUpdater: newMetadataUpdater(func(ctx context.Context, update ...*MetadataUpdate) error { + return store.UpdateAccountsMetadata(ctx, collectionutils.Map(update, func(from *MetadataUpdate) core.Account { + return core.Account{ + Address: from.ID.(string), + Metadata: from.Metadata, } - - effectiveAccounts := make([]core.Account, 0) - for _, account := range ret { - effectiveAccounts = append(effectiveAccounts, account) + })...) + }, 1, 512), + txMetadataUpdater: newMetadataUpdater(func(ctx context.Context, update ...*MetadataUpdate) error { + return store.UpdateTransactionsMetadata(ctx, collectionutils.Map(update, func(from *MetadataUpdate) core.TransactionWithMetadata { + return core.TransactionWithMetadata{ + ID: from.ID.(uint64), + Metadata: from.Metadata, } - - return store.UpdateAccountsMetadata(ctx, effectiveAccounts...) - }, - batching.NoOpOnBatchProcessed[core.Account](), - 1, - 512, - ), - txMetadataWorker: batching.NewBatcher( - store.UpdateTransactionsMetadata, - batching.NoOpOnBatchProcessed[core.TransactionWithMetadata](), - 1, - 512, - ), - moveBuffer: newMoveBuffer(store.InsertMoves, 5, 100), + })...) + }, 1, 512), + moveUpdater: newMoveUpdater(store.InsertMoves, 5, 100), activeLogs: collectionutils.NewLinkedList[*core.ActiveLog](), queue: make(chan []*core.ActiveLog, 1024), stopChan: make(chan chan struct{}), diff --git a/components/ledger/pkg/storage/inmemory.go b/components/ledger/pkg/storage/inmemory.go index 91b0379d72..0cedd69eed 100644 --- a/components/ledger/pkg/storage/inmemory.go +++ b/components/ledger/pkg/storage/inmemory.go @@ -2,6 +2,7 @@ package storage import ( "context" + "sync" "github.com/formancehq/ledger/pkg/core" "github.com/formancehq/stack/libs/go-libs/collectionutils" @@ -9,12 +10,17 @@ import ( ) type InMemoryStore struct { + mu sync.Mutex + Logs []*core.ChainedLog Accounts map[string]*core.AccountWithVolumes Transactions []*core.ExpandedTransaction } func (m *InMemoryStore) MarkedLogsAsProjected(ctx context.Context, id uint64) error { + m.mu.Lock() + defer m.mu.Unlock() + for _, log := range m.Logs { if log.ID == id { log.Projected = true @@ -25,11 +31,15 @@ func (m *InMemoryStore) MarkedLogsAsProjected(ctx context.Context, id uint64) er } func (m *InMemoryStore) InsertMoves(ctx context.Context, insert ...*core.Move) error { + m.mu.Lock() + defer m.mu.Unlock() // TODO(gfyrag): to reflect the behavior of the real storage, we should compute accounts volumes there return nil } func (m *InMemoryStore) UpdateAccountsMetadata(ctx context.Context, update ...core.Account) error { + m.mu.Lock() + defer m.mu.Unlock() for _, account := range update { persistedAccount, ok := m.Accounts[account.Address] if !ok { @@ -45,6 +55,8 @@ func (m *InMemoryStore) UpdateAccountsMetadata(ctx context.Context, update ...co } func (m *InMemoryStore) InsertTransactions(ctx context.Context, insert ...core.Transaction) error { + m.mu.Lock() + defer m.mu.Unlock() for _, transaction := range insert { expandedTransaction := &core.ExpandedTransaction{ Transaction: transaction, @@ -114,6 +126,8 @@ func (m *InMemoryStore) InsertTransactions(ctx context.Context, insert ...core.T } func (m *InMemoryStore) UpdateTransactionsMetadata(ctx context.Context, update ...core.TransactionWithMetadata) error { + m.mu.Lock() + defer m.mu.Unlock() for _, tx := range update { m.Transactions[tx.ID].Metadata = m.Transactions[tx.ID].Metadata.Merge(tx.Metadata) } @@ -121,6 +135,8 @@ func (m *InMemoryStore) UpdateTransactionsMetadata(ctx context.Context, update . } func (m *InMemoryStore) EnsureAccountsExist(ctx context.Context, accounts []string) error { + m.mu.Lock() + defer m.mu.Unlock() for _, address := range accounts { _, ok := m.Accounts[address] if ok { @@ -142,6 +158,8 @@ func (m *InMemoryStore) IsInitialized() bool { } func (m *InMemoryStore) GetNextLogID(ctx context.Context) (uint64, error) { + m.mu.Lock() + defer m.mu.Unlock() for _, log := range m.Logs { if !log.Projected { return log.ID, nil @@ -151,6 +169,9 @@ func (m *InMemoryStore) GetNextLogID(ctx context.Context) (uint64, error) { } func (m *InMemoryStore) ReadLogsRange(ctx context.Context, idMin, idMax uint64) ([]core.ChainedLog, error) { + m.mu.Lock() + defer m.mu.Unlock() + if idMax > uint64(len(m.Logs)) { idMax = uint64(len(m.Logs)) } @@ -165,6 +186,9 @@ func (m *InMemoryStore) ReadLogsRange(ctx context.Context, idMin, idMax uint64) } func (m *InMemoryStore) GetAccountWithVolumes(ctx context.Context, address string) (*core.AccountWithVolumes, error) { + m.mu.Lock() + defer m.mu.Unlock() + account, ok := m.Accounts[address] if !ok { return &core.AccountWithVolumes{ @@ -179,6 +203,9 @@ func (m *InMemoryStore) GetAccountWithVolumes(ctx context.Context, address strin } func (m *InMemoryStore) GetTransaction(ctx context.Context, id uint64) (*core.ExpandedTransaction, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.Transactions[id], nil } From 175d7577361f1f33f78f1e8640b115cda8d28fe2 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Mon, 17 Jul 2023 01:22:53 +0200 Subject: [PATCH 7/7] fix: missing moves in transactions migration --- .../pkg/storage/ledgerstore/migrates/0-init-schema/any.go | 6 ++++++ tests/integration/suite/ledger-list-count-accounts.go | 1 + 2 files changed, 7 insertions(+) diff --git a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go index 75b9caa4de..cebe581865 100644 --- a/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go +++ b/components/ledger/pkg/storage/ledgerstore/migrates/0-init-schema/any.go @@ -251,6 +251,9 @@ func migrateLogs( if err := store.InsertTransactions(ctx, *payload.Transaction); err != nil { return err } + if err := store.InsertMoves(ctx, payload.Transaction.GetMoves()...); err != nil { + return err + } case core.SetMetadataLogPayload: switch payload.TargetType { case core.MetaTargetTypeTransaction: @@ -272,6 +275,9 @@ func migrateLogs( if err := store.InsertTransactions(ctx, *payload.RevertTransaction); err != nil { return err } + if err := store.InsertMoves(ctx, payload.RevertTransaction.GetMoves()...); err != nil { + return err + } if err := store.UpdateTransactionsMetadata(ctx, core.TransactionWithMetadata{ ID: payload.RevertedTransactionID, Metadata: core.RevertedMetadata(payload.RevertTransaction.ID), diff --git a/tests/integration/suite/ledger-list-count-accounts.go b/tests/integration/suite/ledger-list-count-accounts.go index 4313e38b79..a95881b32d 100644 --- a/tests/integration/suite/ledger-list-count-accounts.go +++ b/tests/integration/suite/ledger-list-count-accounts.go @@ -2,6 +2,7 @@ package suite import ( "fmt" + "math/big" "sort" "time"