diff --git a/go.mod b/go.mod index ae74083e8..d321b71cd 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( github.com/shomali11/xsql v0.0.0-20190608141458-bf76292144df github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 + github.com/stoewer/go-strcase v1.3.0 github.com/stretchr/testify v1.10.0 github.com/uptrace/bun v1.2.6 github.com/uptrace/bun/dialect/pgdialect v1.2.6 diff --git a/go.sum b/go.sum index b526814ed..103fec330 100644 --- a/go.sum +++ b/go.sum @@ -316,6 +316,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/internal/README.md b/internal/README.md index debde2f3e..42282d7d3 100644 --- a/internal/README.md +++ b/internal/README.md @@ -15,6 +15,7 @@ import "github.com/formancehq/ledger/internal" - [func \(a Account\) GetAddress\(\) string](<#Account.GetAddress>) - [type AccountMetadata](<#AccountMetadata>) - [type AccountsVolumes](<#AccountsVolumes>) +- [type AggregatedVolumes](<#AggregatedVolumes>) - [type BalancesByAssets](<#BalancesByAssets>) - [type BalancesByAssetsByAccounts](<#BalancesByAssetsByAccounts>) - [type Configuration](<#Configuration>) @@ -207,6 +208,17 @@ type AccountsVolumes struct { } ``` + +## type AggregatedVolumes + + + +```go +type AggregatedVolumes struct { + Aggregated VolumesByAssets `bun:"aggregated,type:jsonb"` +} +``` + ## type BalancesByAssets diff --git a/internal/api/bulking/mocks_ledger_controller_test.go b/internal/api/bulking/mocks_ledger_controller_test.go index 7786df0be..2cede2100 100644 --- a/internal/api/bulking/mocks_ledger_controller_test.go +++ b/internal/api/bulking/mocks_ledger_controller_test.go @@ -70,7 +70,7 @@ func (mr *LedgerControllerMockRecorder) Commit(ctx any) *gomock.Call { } // CountAccounts mocks base method. -func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (int, error) { +func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountAccounts", ctx, query) ret0, _ := ret[0].(int) @@ -85,7 +85,7 @@ func (mr *LedgerControllerMockRecorder) CountAccounts(ctx, query any) *gomock.Ca } // CountTransactions mocks base method. -func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (int, error) { +func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountTransactions", ctx, query) ret0, _ := ret[0].(int) @@ -160,7 +160,7 @@ func (mr *LedgerControllerMockRecorder) Export(ctx, w any) *gomock.Call { } // GetAccount mocks base method. -func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.GetAccountQuery) (*ledger.Account, error) { +func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Account, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAccount", ctx, query) ret0, _ := ret[0].(*ledger.Account) @@ -175,7 +175,7 @@ func (mr *LedgerControllerMockRecorder) GetAccount(ctx, query any) *gomock.Call } // GetAggregatedBalances mocks base method. -func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { +func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.ResourceQuery[ledger0.GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAggregatedBalances", ctx, q) ret0, _ := ret[0].(ledger.BalancesByAssets) @@ -220,7 +220,7 @@ func (mr *LedgerControllerMockRecorder) GetStats(ctx any) *gomock.Call { } // GetTransaction mocks base method. -func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.GetTransactionQuery) (*ledger.Transaction, error) { +func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Transaction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTransaction", ctx, query) ret0, _ := ret[0].(*ledger.Transaction) @@ -235,7 +235,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -279,7 +279,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -294,7 +294,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -309,7 +309,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/api/common/mocks_ledger_controller_test.go b/internal/api/common/mocks_ledger_controller_test.go index 85e72e1b8..c263cfa1f 100644 --- a/internal/api/common/mocks_ledger_controller_test.go +++ b/internal/api/common/mocks_ledger_controller_test.go @@ -70,7 +70,7 @@ func (mr *LedgerControllerMockRecorder) Commit(ctx any) *gomock.Call { } // CountAccounts mocks base method. -func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (int, error) { +func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountAccounts", ctx, query) ret0, _ := ret[0].(int) @@ -85,7 +85,7 @@ func (mr *LedgerControllerMockRecorder) CountAccounts(ctx, query any) *gomock.Ca } // CountTransactions mocks base method. -func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (int, error) { +func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountTransactions", ctx, query) ret0, _ := ret[0].(int) @@ -160,7 +160,7 @@ func (mr *LedgerControllerMockRecorder) Export(ctx, w any) *gomock.Call { } // GetAccount mocks base method. -func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.GetAccountQuery) (*ledger.Account, error) { +func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Account, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAccount", ctx, query) ret0, _ := ret[0].(*ledger.Account) @@ -175,7 +175,7 @@ func (mr *LedgerControllerMockRecorder) GetAccount(ctx, query any) *gomock.Call } // GetAggregatedBalances mocks base method. -func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { +func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.ResourceQuery[ledger0.GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAggregatedBalances", ctx, q) ret0, _ := ret[0].(ledger.BalancesByAssets) @@ -220,7 +220,7 @@ func (mr *LedgerControllerMockRecorder) GetStats(ctx any) *gomock.Call { } // GetTransaction mocks base method. -func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.GetTransactionQuery) (*ledger.Transaction, error) { +func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Transaction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTransaction", ctx, query) ret0, _ := ret[0].(*ledger.Transaction) @@ -235,7 +235,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -279,7 +279,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -294,7 +294,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -309,7 +309,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/api/v1/controllers_accounts_count.go b/internal/api/v1/controllers_accounts_count.go index 9a5bb94be..b6efe8ddd 100644 --- a/internal/api/v1/controllers_accounts_count.go +++ b/internal/api/v1/controllers_accounts_count.go @@ -6,8 +6,6 @@ import ( "errors" "github.com/formancehq/go-libs/v2/api" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" - "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) @@ -15,23 +13,19 @@ import ( func countAccounts(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query, err := bunpaginate.Extract[ledgercontroller.ListAccountsQuery](r, func() (*ledgercontroller.ListAccountsQuery, error) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) - if err != nil { - return nil, err - } - options.QueryBuilder, err = buildAccountsFilterQuery(r) - if err != nil { - return nil, err - } - return pointer.For(ledgercontroller.NewListAccountsQuery(*options)), nil - }) + rq, err := getResourceQuery[any](r) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } + + rq.Builder, err = buildAccountsFilterQuery(r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - count, err := l.CountAccounts(r.Context(), *query) + count, err := l.CountAccounts(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v1/controllers_accounts_count_test.go b/internal/api/v1/controllers_accounts_count_test.go index 1ebea8349..3ce9e9416 100644 --- a/internal/api/v1/controllers_accounts_count_test.go +++ b/internal/api/v1/controllers_accounts_count_test.go @@ -5,7 +5,7 @@ import ( "net/http" "net/http/httptest" "net/url" - os "os" + "os" "testing" "errors" @@ -24,7 +24,7 @@ func TestAccountsCount(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + expectQuery ledgercontroller.ResourceQuery[any] expectStatusCode int expectedErrorCode string returnErr error @@ -34,13 +34,8 @@ func TestAccountsCount(t *testing.T) { testCases := []testCase{ { - name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + name: "nominal", + expectQuery: ledgercontroller.ResourceQuery[any]{}, expectBackendCall: true, }, { @@ -49,33 +44,17 @@ func TestAccountsCount(t *testing.T) { "metadata[roles]": []string{"admin"}, }, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("metadata[roles]", "admin")). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("metadata[roles]", "admin"), + }, }, { name: "using address", queryParams: url.Values{"address": []string{"foo"}}, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("address", "foo")). - WithPageSize(DefaultPageSize), - }, - { - name: "invalid page size", - queryParams: url.Values{ - "pageSize": []string{"nan"}, + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "foo"), }, - expectStatusCode: http.StatusBadRequest, - expectedErrorCode: common.ErrValidation, }, { name: "page size over maximum", @@ -83,12 +62,7 @@ func TestAccountsCount(t *testing.T) { queryParams: url.Values{ "pageSize": []string{"1000000"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(MaxPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{}, }, { name: "using balance filter", @@ -97,13 +71,9 @@ func TestAccountsCount(t *testing.T) { "balance": []string{"100"}, }, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Lt("balance", int64(100))). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("balance", int64(100)), + }, }, { name: "with invalid query from core point of view", @@ -111,12 +81,7 @@ func TestAccountsCount(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrInvalidQuery{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{}, }, { name: "with missing feature", @@ -124,12 +89,7 @@ func TestAccountsCount(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{}, }, { name: "with unexpected error", @@ -137,12 +97,7 @@ func TestAccountsCount(t *testing.T) { expectedErrorCode: api.ErrorInternal, expectBackendCall: true, returnErr: errors.New("undefined error"), - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{}, }, } for _, testCase := range testCases { @@ -156,7 +111,7 @@ func TestAccountsCount(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectBackendCall { ledgerController.EXPECT(). - CountAccounts(gomock.Any(), ledgercontroller.NewListAccountsQuery(testCase.expectQuery)). + CountAccounts(gomock.Any(), testCase.expectQuery). Return(10, testCase.returnErr) } diff --git a/internal/api/v1/controllers_accounts_list.go b/internal/api/v1/controllers_accounts_list.go index 9c1c29bac..6b358547f 100644 --- a/internal/api/v1/controllers_accounts_list.go +++ b/internal/api/v1/controllers_accounts_list.go @@ -5,8 +5,6 @@ import ( "errors" "github.com/formancehq/go-libs/v2/api" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" - "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) @@ -14,23 +12,19 @@ import ( func listAccounts(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query, err := bunpaginate.Extract[ledgercontroller.ListAccountsQuery](r, func() (*ledgercontroller.ListAccountsQuery, error) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) - if err != nil { - return nil, err - } - options.QueryBuilder, err = buildAccountsFilterQuery(r) - if err != nil { - return nil, err - } - return pointer.For(ledgercontroller.NewListAccountsQuery(*options)), nil - }) + rq, err := getOffsetPaginatedQuery[any](r) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } + + rq.Options.Builder, err = buildAccountsFilterQuery(r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - cursor, err := l.ListAccounts(r.Context(), *query) + cursor, err := l.ListAccounts(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v1/controllers_accounts_list_test.go b/internal/api/v1/controllers_accounts_list_test.go index 8f1677953..2784c4df0 100644 --- a/internal/api/v1/controllers_accounts_list_test.go +++ b/internal/api/v1/controllers_accounts_list_test.go @@ -25,7 +25,7 @@ func TestAccountsList(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + expectQuery ledgercontroller.OffsetPaginatedQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -36,8 +36,9 @@ func TestAccountsList(t *testing.T) { { name: "nominal", expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + }, }, { name: "using metadata", @@ -45,9 +46,12 @@ func TestAccountsList(t *testing.T) { "metadata[roles]": []string{"admin"}, }, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("metadata[roles]", "admin")). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("metadata[roles]", "admin"), + }, + }, }, { name: "using address", @@ -55,17 +59,20 @@ func TestAccountsList(t *testing.T) { "address": []string{"foo"}, }, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("address", "foo")). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "foo"), + }, + }, }, { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{})))}, + "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.OffsetPaginatedQuery[any]{})}, }, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}), + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{}, }, { name: "using invalid cursor", @@ -89,8 +96,9 @@ func TestAccountsList(t *testing.T) { "pageSize": []string{"1000000"}, }, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithPageSize(MaxPageSize), + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: MaxPageSize, + }, }, { name: "using balance filter", @@ -99,9 +107,12 @@ func TestAccountsList(t *testing.T) { "balanceOperator": []string{"e"}, }, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("balance", int64(100))). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("balance", int64(100)), + }, + }, }, { name: "with missing feature", @@ -109,8 +120,9 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: common.ErrValidation, returnErr: ledgercontroller.ErrMissingFeature{}, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + }, }, } for _, testCase := range testCases { @@ -133,7 +145,7 @@ func TestAccountsList(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectBackendCall { ledgerController.EXPECT(). - ListAccounts(gomock.Any(), ledgercontroller.NewListAccountsQuery(testCase.expectQuery)). + ListAccounts(gomock.Any(), testCase.expectQuery). Return(&expectedCursor, testCase.returnErr) } diff --git a/internal/api/v1/controllers_accounts_read.go b/internal/api/v1/controllers_accounts_read.go index 3a56a9f66..9b0ebe209 100644 --- a/internal/api/v1/controllers_accounts_read.go +++ b/internal/api/v1/controllers_accounts_read.go @@ -1,6 +1,7 @@ package v1 import ( + "github.com/formancehq/go-libs/v2/query" "net/http" "net/url" @@ -22,10 +23,10 @@ func getAccount(w http.ResponseWriter, r *http.Request) { return } - query := ledgercontroller.NewGetAccountQuery(address) - query = query.WithExpandVolumes() - - acc, err := l.GetAccount(r.Context(), query) + acc, err := l.GetAccount(r.Context(), ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", address), + Expand: []string{"volumes"}, + }) if err != nil { switch { case postgres.IsNotFoundError(err): diff --git a/internal/api/v1/controllers_accounts_read_test.go b/internal/api/v1/controllers_accounts_read_test.go index 027415873..1a57dc407 100644 --- a/internal/api/v1/controllers_accounts_read_test.go +++ b/internal/api/v1/controllers_accounts_read_test.go @@ -2,6 +2,7 @@ package v1 import ( "bytes" + "github.com/formancehq/go-libs/v2/query" "github.com/formancehq/ledger/internal/api/common" "net/http" "net/http/httptest" @@ -25,7 +26,7 @@ func TestAccountsRead(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.GetAccountQuery + expectQuery ledgercontroller.ResourceQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -35,15 +36,21 @@ func TestAccountsRead(t *testing.T) { testCases := []testCase{ { - name: "nominal", - account: "foo", - expectQuery: ledgercontroller.NewGetAccountQuery("foo").WithExpandVolumes(), + name: "nominal", + account: "foo", + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "foo"), + Expand: []string{"volumes"}, + }, expectBackendCall: true, }, { - name: "with expand volumes", - account: "foo", - expectQuery: ledgercontroller.NewGetAccountQuery("foo").WithExpandVolumes(), + name: "with expand volumes", + account: "foo", + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "foo"), + Expand: []string{"volumes"}, + }, expectBackendCall: true, queryParams: url.Values{ "expand": {"volumes"}, @@ -56,9 +63,12 @@ func TestAccountsRead(t *testing.T) { expectedErrorCode: common.ErrValidation, }, { - name: "with not existing account", - account: "foo", - expectQuery: ledgercontroller.NewGetAccountQuery("foo").WithExpandVolumes(), + name: "with not existing account", + account: "foo", + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "foo"), + Expand: []string{"volumes"}, + }, expectBackendCall: true, returnErr: postgres.ErrNotFound, }, diff --git a/internal/api/v1/controllers_balances_aggregates.go b/internal/api/v1/controllers_balances_aggregates.go index dd0147346..135a5791a 100644 --- a/internal/api/v1/controllers_balances_aggregates.go +++ b/internal/api/v1/controllers_balances_aggregates.go @@ -18,20 +18,19 @@ func buildAggregatedBalancesQuery(r *http.Request) query.Builder { } func getBalancesAggregated(w http.ResponseWriter, r *http.Request) { + rq, err := getResourceQuery[ledgercontroller.GetAggregatedVolumesOptions](r, func(q *ledgercontroller.GetAggregatedVolumesOptions) error { + q.UseInsertionDate = true - pitFilter, err := getPITFilter(r) + return nil + }) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - queryBuilder := buildAggregatedBalancesQuery(r) + rq.Builder = buildAggregatedBalancesQuery(r) - query := ledgercontroller.NewGetAggregatedBalancesQuery(*pitFilter, queryBuilder, - // notes(gfyrag): if pit is not specified, always use insertion date to be backward compatible - r.URL.Query().Get("pit") == "" || api.QueryParamBool(r, "useInsertionDate") || api.QueryParamBool(r, "use_insertion_date")) - - balances, err := common.LedgerFromContext(r.Context()).GetAggregatedBalances(r.Context(), query) + balances, err := common.LedgerFromContext(r.Context()).GetAggregatedBalances(r.Context(), *rq) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_balances_aggregates_test.go b/internal/api/v1/controllers_balances_aggregates_test.go index 18e0f535a..7da73d388 100644 --- a/internal/api/v1/controllers_balances_aggregates_test.go +++ b/internal/api/v1/controllers_balances_aggregates_test.go @@ -10,8 +10,6 @@ import ( ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/go-libs/v2/time" - "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/go-libs/v2/auth" "github.com/formancehq/go-libs/v2/query" @@ -26,16 +24,16 @@ func TestBalancesAggregates(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery ledgercontroller.GetAggregatedBalanceQuery + expectQuery ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions] } - now := time.Now() - testCases := []testCase{ { name: "nominal", - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - UseInsertionDate: true, + expectQuery: ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{ + UseInsertionDate: true, + }, }, }, { @@ -43,33 +41,11 @@ func TestBalancesAggregates(t *testing.T) { queryParams: url.Values{ "address": []string{"foo"}, }, - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - QueryBuilder: query.Match("address", "foo"), - UseInsertionDate: true, - }, - }, - { - name: "using pit", - queryParams: url.Values{ - "pit": []string{now.Format(time.RFC3339Nano)}, - }, - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, - }, - }, - }, - { - name: "using pit + insertion date", - queryParams: url.Values{ - "pit": []string{now.Format(time.RFC3339Nano)}, - "useInsertionDate": []string{"true"}, - }, - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{ + UseInsertionDate: true, }, - UseInsertionDate: true, + Builder: query.Match("address", "foo"), }, }, } diff --git a/internal/api/v1/controllers_balances_list.go b/internal/api/v1/controllers_balances_list.go index fb7217e15..397e1bb93 100644 --- a/internal/api/v1/controllers_balances_list.go +++ b/internal/api/v1/controllers_balances_list.go @@ -6,31 +6,26 @@ import ( "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/go-libs/v2/bun/bunpaginate" - "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) func getBalances(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - q, err := bunpaginate.Extract[ledgercontroller.ListAccountsQuery](r, func() (*ledgercontroller.ListAccountsQuery, error) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) - if err != nil { - return nil, err - } - options.QueryBuilder, err = buildAccountsFilterQuery(r) - if err != nil { - return nil, err - } - return pointer.For(ledgercontroller.NewListAccountsQuery(*options)), nil - }) + rq, err := getOffsetPaginatedQuery[any](r) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } + + rq.Options.Builder, err = buildAccountsFilterQuery(r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } + rq.Options.Expand = []string{"volumes"} - cursor, err := l.ListAccounts(r.Context(), q.WithExpandVolumes()) + cursor, err := l.ListAccounts(r.Context(), *rq) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_logs_list.go b/internal/api/v1/controllers_logs_list.go index 1fd7d1777..ca4d41ae5 100644 --- a/internal/api/v1/controllers_logs_list.go +++ b/internal/api/v1/controllers_logs_list.go @@ -1,14 +1,12 @@ package v1 import ( - "fmt" "net/http" "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/query" "github.com/formancehq/ledger/internal/api/common" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) func buildGetLogsQuery(r *http.Request) query.Builder { @@ -37,32 +35,15 @@ func buildGetLogsQuery(r *http.Request) query.Builder { func getLogs(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query := ledgercontroller.GetLogsQuery{} - - if r.URL.Query().Get(QueryKeyCursor) != "" { - err := bunpaginate.UnmarshalCursor(r.URL.Query().Get(QueryKeyCursor), &query) - if err != nil { - api.BadRequest(w, common.ErrValidation, fmt.Errorf("invalid '%s' query param: %w", QueryKeyCursor, err)) - return - } - } else { - var err error - - pageSize, err := bunpaginate.GetPageSize(r, - bunpaginate.WithDefaultPageSize(DefaultPageSize), - bunpaginate.WithMaxPageSize(MaxPageSize)) - if err != nil { - common.HandleCommonErrors(w, r, err) - return - } - - query = ledgercontroller.NewListLogsQuery(ledgercontroller.PaginatedQueryOptions[any]{ - QueryBuilder: buildGetLogsQuery(r), - PageSize: pageSize, - }) + paginatedQuery, err := getColumnPaginatedQuery[any](r, "id", bunpaginate.OrderDesc) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return } - cursor, err := l.ListLogs(r.Context(), query) + paginatedQuery.Options.Builder = buildGetLogsQuery(r) + + cursor, err := l.ListLogs(r.Context(), *paginatedQuery) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_logs_list_test.go b/internal/api/v1/controllers_logs_list_test.go index 44a419535..20d381db6 100644 --- a/internal/api/v1/controllers_logs_list_test.go +++ b/internal/api/v1/controllers_logs_list_test.go @@ -2,6 +2,7 @@ package v1 import ( "encoding/json" + "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" "net/http" "net/http/httptest" @@ -26,7 +27,7 @@ func TestGetLogs(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery ledgercontroller.PaginatedQueryOptions[any] + expectQuery ledgercontroller.ColumnPaginatedQuery[any] expectStatusCode int expectedErrorCode string } @@ -34,30 +35,47 @@ func TestGetLogs(t *testing.T) { now := time.Now() testCases := []testCase{ { - name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil), + name: "nominal", + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + }, }, { name: "using start time", queryParams: url.Values{ "start_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil).WithQueryBuilder(query.Gte("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Gte("date", now.Format(time.DateFormat)), + }, + }, }, { name: "using end time", queryParams: url.Values{ "end_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil). - WithQueryBuilder(query.Lt("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("date", now.Format(time.DateFormat)), + }, + }, }, { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil)))}, + "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.ColumnPaginatedQuery[any]{})}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{}, }, { name: "using invalid cursor", @@ -88,7 +106,7 @@ func TestGetLogs(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { ledgerController.EXPECT(). - ListLogs(gomock.Any(), ledgercontroller.NewListLogsQuery(testCase.expectQuery)). + ListLogs(gomock.Any(), testCase.expectQuery). Return(&expectedCursor, nil) } diff --git a/internal/api/v1/controllers_transactions_count.go b/internal/api/v1/controllers_transactions_count.go index a36e009f9..d67ec9f01 100644 --- a/internal/api/v1/controllers_transactions_count.go +++ b/internal/api/v1/controllers_transactions_count.go @@ -6,20 +6,17 @@ import ( "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/ledger/internal/api/common" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) func countTransactions(w http.ResponseWriter, r *http.Request) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) + rq, err := getResourceQuery[any](r) if err != nil { - api.BadRequest(w, common.ErrValidation, err) return } - options.QueryBuilder = buildGetTransactionsQuery(r) + rq.Builder = buildGetTransactionsQuery(r) - count, err := common.LedgerFromContext(r.Context()). - CountTransactions(r.Context(), ledgercontroller.NewListTransactionsQuery(*options)) + count, err := common.LedgerFromContext(r.Context()).CountTransactions(r.Context(), *rq) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_transactions_count_test.go b/internal/api/v1/controllers_transactions_count_test.go index 1985abe49..805c611b0 100644 --- a/internal/api/v1/controllers_transactions_count_test.go +++ b/internal/api/v1/controllers_transactions_count_test.go @@ -22,7 +22,7 @@ func TestCountTransactions(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + expectQuery ledgercontroller.ResourceQuery[any] expectStatusCode int expectedErrorCode string } @@ -31,63 +31,70 @@ func TestCountTransactions(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}), + expectQuery: ledgercontroller.ResourceQuery[any]{}, }, { name: "using metadata", queryParams: url.Values{ "metadata[roles]": []string{"admin"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("metadata[roles]", "admin")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("metadata[roles]", "admin"), + }, }, { name: "using startTime", queryParams: url.Values{ "start_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Gte("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Gte("date", now.Format(time.DateFormat)), + }, }, { name: "using endTime", queryParams: url.Values{ "end_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Lt("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("date", now.Format(time.DateFormat)), + }, }, { name: "using account", queryParams: url.Values{ "account": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("account", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("account", "xxx"), + }, }, { name: "using reference", queryParams: url.Values{ "reference": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("reference", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("reference", "xxx"), + }, }, { name: "using destination", queryParams: url.Values{ "destination": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("destination", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("destination", "xxx"), + }, }, { name: "using source", queryParams: url.Values{ "source": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("source", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("source", "xxx"), + }, }, } for _, testCase := range testCases { @@ -101,7 +108,7 @@ func TestCountTransactions(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { ledgerController.EXPECT(). - CountTransactions(gomock.Any(), ledgercontroller.NewListTransactionsQuery(testCase.expectQuery)). + CountTransactions(gomock.Any(), testCase.expectQuery). Return(10, nil) } diff --git a/internal/api/v1/controllers_transactions_list.go b/internal/api/v1/controllers_transactions_list.go index ae1c11fa8..bf575cb8c 100644 --- a/internal/api/v1/controllers_transactions_list.go +++ b/internal/api/v1/controllers_transactions_list.go @@ -5,29 +5,21 @@ import ( "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/go-libs/v2/bun/bunpaginate" - "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) func listTransactions(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query, err := bunpaginate.Extract[ledgercontroller.ListTransactionsQuery](r, func() (*ledgercontroller.ListTransactionsQuery, error) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) - if err != nil { - return nil, err - } - options.QueryBuilder = buildGetTransactionsQuery(r) - - return pointer.For(ledgercontroller.NewListTransactionsQuery(*options)), nil - }) + paginatedQuery, err := getColumnPaginatedQuery[any](r, "id", bunpaginate.OrderDesc) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } + paginatedQuery.Options.Builder = buildGetTransactionsQuery(r) + paginatedQuery.Options.Expand = []string{"volumes"} - cursor, err := l.ListTransactions(r.Context(), *query) + cursor, err := l.ListTransactions(r.Context(), *paginatedQuery) if err != nil { common.HandleCommonErrors(w, r, err) return diff --git a/internal/api/v1/controllers_transactions_list_test.go b/internal/api/v1/controllers_transactions_list_test.go index 45bbaf2cc..e8aa5cf75 100644 --- a/internal/api/v1/controllers_transactions_list_test.go +++ b/internal/api/v1/controllers_transactions_list_test.go @@ -1,6 +1,7 @@ package v1 import ( + "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" "math/big" "net/http" @@ -26,7 +27,7 @@ func TestTransactionsList(t *testing.T) { type testCase struct { name string queryParams url.Values - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + expectQuery ledgercontroller.ColumnPaginatedQuery[any] expectStatusCode int expectedErrorCode string } @@ -34,71 +35,131 @@ func TestTransactionsList(t *testing.T) { testCases := []testCase{ { - name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}), + name: "nominal", + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Expand: []string{"volumes"}, + }, + }, }, { name: "using metadata", queryParams: url.Values{ "metadata[roles]": []string{"admin"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("metadata[roles]", "admin")), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("metadata[roles]", "admin"), + Expand: []string{"volumes"}, + }, + }, }, { name: "using startTime", queryParams: url.Values{ "start_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Gte("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Gte("date", now.Format(time.DateFormat)), + Expand: []string{"volumes"}, + }, + }, }, { name: "using endTime", queryParams: url.Values{ "end_time": []string{now.Format(time.DateFormat)}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Lt("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("date", now.Format(time.DateFormat)), + Expand: []string{"volumes"}, + }, + }, }, { name: "using account", queryParams: url.Values{ "account": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("account", "xxx")), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("account", "xxx"), + Expand: []string{"volumes"}, + }, + }, }, { name: "using reference", queryParams: url.Values{ "reference": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("reference", "xxx")), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("reference", "xxx"), + Expand: []string{"volumes"}, + }, + }, }, { name: "using destination", queryParams: url.Values{ "destination": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("destination", "xxx")), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("destination", "xxx"), + Expand: []string{"volumes"}, + }, + }, }, { name: "using source", queryParams: url.Values{ "source": []string{"xxx"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("source", "xxx")), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("source", "xxx"), + Expand: []string{"volumes"}, + }, + }, }, { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{})))}, + "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.ColumnPaginatedQuery[any]{})}, + }, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Expand: []string{"volumes"}, + }, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}), }, { name: "using invalid cursor", @@ -121,8 +182,14 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "pageSize": []string{"1000000"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithPageSize(MaxPageSize), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: MaxPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + Options: ledgercontroller.ResourceQuery[any]{ + Expand: []string{"volumes"}, + }, + }, }, } for _, testCase := range testCases { @@ -144,7 +211,7 @@ func TestTransactionsList(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { ledgerController.EXPECT(). - ListTransactions(gomock.Any(), ledgercontroller.NewListTransactionsQuery(testCase.expectQuery)). + ListTransactions(gomock.Any(), testCase.expectQuery). Return(&expectedCursor, nil) } diff --git a/internal/api/v1/controllers_transactions_read.go b/internal/api/v1/controllers_transactions_read.go index d1b2dd147..b2102ed88 100644 --- a/internal/api/v1/controllers_transactions_read.go +++ b/internal/api/v1/controllers_transactions_read.go @@ -1,14 +1,13 @@ package v1 import ( + "github.com/formancehq/go-libs/v2/query" "net/http" "strconv" "github.com/formancehq/go-libs/v2/api" - "github.com/formancehq/go-libs/v2/collectionutils" "github.com/formancehq/go-libs/v2/platform/postgres" "github.com/formancehq/ledger/internal/api/common" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/go-chi/chi/v5" ) @@ -21,15 +20,14 @@ func readTransaction(w http.ResponseWriter, r *http.Request) { return } - query := ledgercontroller.NewGetTransactionQuery(int(txId)) - if collectionutils.Contains(r.URL.Query()["expand"], "volumes") { - query = query.WithExpandVolumes() - } - if collectionutils.Contains(r.URL.Query()["expand"], "effectiveVolumes") { - query = query.WithExpandEffectiveVolumes() + rq, err := getResourceQuery[any](r) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return } + rq.Builder = query.Match("id", txId) - tx, err := l.GetTransaction(r.Context(), query) + tx, err := l.GetTransaction(r.Context(), *rq) if err != nil { switch { case postgres.IsNotFoundError(err): diff --git a/internal/api/v1/controllers_transactions_read_test.go b/internal/api/v1/controllers_transactions_read_test.go index 7369ff644..d3e683773 100644 --- a/internal/api/v1/controllers_transactions_read_test.go +++ b/internal/api/v1/controllers_transactions_read_test.go @@ -1,6 +1,7 @@ package v1 import ( + "github.com/formancehq/go-libs/v2/query" "math/big" "net/http" "net/http/httptest" @@ -24,7 +25,9 @@ func TestTransactionsRead(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) ledgerController.EXPECT(). - GetTransaction(gomock.Any(), ledgercontroller.NewGetTransactionQuery(0)). + GetTransaction(gomock.Any(), ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", int64(0)), + }). Return(&tx, nil) router := NewRouter(systemController, auth.NewNoAuth(), "develop", os.Getenv("DEBUG") == "true") diff --git a/internal/api/v1/mocks_ledger_controller_test.go b/internal/api/v1/mocks_ledger_controller_test.go index e619609c9..f89439826 100644 --- a/internal/api/v1/mocks_ledger_controller_test.go +++ b/internal/api/v1/mocks_ledger_controller_test.go @@ -70,7 +70,7 @@ func (mr *LedgerControllerMockRecorder) Commit(ctx any) *gomock.Call { } // CountAccounts mocks base method. -func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (int, error) { +func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountAccounts", ctx, query) ret0, _ := ret[0].(int) @@ -85,7 +85,7 @@ func (mr *LedgerControllerMockRecorder) CountAccounts(ctx, query any) *gomock.Ca } // CountTransactions mocks base method. -func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (int, error) { +func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountTransactions", ctx, query) ret0, _ := ret[0].(int) @@ -160,7 +160,7 @@ func (mr *LedgerControllerMockRecorder) Export(ctx, w any) *gomock.Call { } // GetAccount mocks base method. -func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.GetAccountQuery) (*ledger.Account, error) { +func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Account, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAccount", ctx, query) ret0, _ := ret[0].(*ledger.Account) @@ -175,7 +175,7 @@ func (mr *LedgerControllerMockRecorder) GetAccount(ctx, query any) *gomock.Call } // GetAggregatedBalances mocks base method. -func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { +func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.ResourceQuery[ledger0.GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAggregatedBalances", ctx, q) ret0, _ := ret[0].(ledger.BalancesByAssets) @@ -220,7 +220,7 @@ func (mr *LedgerControllerMockRecorder) GetStats(ctx any) *gomock.Call { } // GetTransaction mocks base method. -func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.GetTransactionQuery) (*ledger.Transaction, error) { +func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Transaction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTransaction", ctx, query) ret0, _ := ret[0].(*ledger.Transaction) @@ -235,7 +235,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -279,7 +279,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -294,7 +294,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -309,7 +309,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/api/v1/utils.go b/internal/api/v1/utils.go index f94d5cb07..c6e2d3242 100644 --- a/internal/api/v1/utils.go +++ b/internal/api/v1/utils.go @@ -6,66 +6,11 @@ import ( ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/go-libs/v2/time" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" - "github.com/formancehq/go-libs/v2/collectionutils" "github.com/formancehq/go-libs/v2/pointer" - "github.com/formancehq/go-libs/v2/query" ) -func getPITFilter(r *http.Request) (*ledgercontroller.PITFilter, error) { - pitString := r.URL.Query().Get("pit") - if pitString == "" { - return &ledgercontroller.PITFilter{}, nil - } - pit, err := time.ParseTime(pitString) - if err != nil { - return nil, err - } - return &ledgercontroller.PITFilter{ - PIT: &pit, - }, nil -} - -func getPITFilterWithVolumes(r *http.Request) (*ledgercontroller.PITFilterWithVolumes, error) { - pit, err := getPITFilter(r) - if err != nil { - return nil, err - } - return &ledgercontroller.PITFilterWithVolumes{ - PITFilter: *pit, - ExpandVolumes: collectionutils.Contains(r.URL.Query()["expand"], "volumes"), - ExpandEffectiveVolumes: collectionutils.Contains(r.URL.Query()["expand"], "effectiveVolumes"), - }, nil -} - -func getQueryBuilder(r *http.Request) (query.Builder, error) { - return query.ParseJSON(r.URL.Query().Get("query")) -} - -func getPaginatedQueryOptionsOfPITFilterWithVolumes(r *http.Request) (*ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes], error) { - qb, err := getQueryBuilder(r) - if err != nil { - return nil, err - } - - pitFilter, err := getPITFilterWithVolumes(r) - if err != nil { - return nil, err - } - - pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) - if err != nil { - return nil, err - } - - return pointer.For(ledgercontroller.NewPaginatedQueryOptions(*pitFilter). - WithQueryBuilder(qb). - WithPageSize(pageSize)), nil -} - func getCommandParameters[INPUT any](r *http.Request, input INPUT) ledgercontroller.Parameters[INPUT] { dryRunAsString := r.URL.Query().Get("preview") dryRun := strings.ToUpper(dryRunAsString) == "YES" || strings.ToUpper(dryRunAsString) == "TRUE" || dryRunAsString == "1" @@ -78,3 +23,57 @@ func getCommandParameters[INPUT any](r *http.Request, input INPUT) ledgercontrol Input: input, } } + +func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) { + return bunpaginate.Extract[ledgercontroller.OffsetPaginatedQuery[v]](r, func() (*ledgercontroller.OffsetPaginatedQuery[v], error) { + rq, err := getResourceQuery[v](r, modifiers...) + if err != nil { + return nil, err + } + + pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) + if err != nil { + return nil, err + } + + return &ledgercontroller.OffsetPaginatedQuery[v]{ + PageSize: pageSize, + Options: *rq, + }, nil + }) +} + +func getColumnPaginatedQuery[v any](r *http.Request, column string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) { + return bunpaginate.Extract[ledgercontroller.ColumnPaginatedQuery[v]](r, func() (*ledgercontroller.ColumnPaginatedQuery[v], error) { + rq, err := getResourceQuery[v](r, modifiers...) + if err != nil { + return nil, err + } + + pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) + if err != nil { + return nil, err + } + + return &ledgercontroller.ColumnPaginatedQuery[v]{ + PageSize: pageSize, + Column: column, + Order: pointer.For(order), + Options: *rq, + }, nil + }) +} + +func getResourceQuery[v any](r *http.Request, modifiers ...func(*v) error) (*ledgercontroller.ResourceQuery[v], error) { + var options v + for _, modifier := range modifiers { + if err := modifier(&options); err != nil { + return nil, err + } + } + + return &ledgercontroller.ResourceQuery[v]{ + Expand: r.URL.Query()["expand"], + Opts: options, + }, nil +} diff --git a/internal/api/v2/common.go b/internal/api/v2/common.go index b223f1fee..6d7c94185 100644 --- a/internal/api/v2/common.go +++ b/internal/api/v2/common.go @@ -1,16 +1,12 @@ package v2 import ( + . "github.com/formancehq/go-libs/v2/collectionutils" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "io" "net/http" - "slices" - "strconv" "strings" - "github.com/formancehq/go-libs/v2/api" - - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/time" @@ -18,106 +14,27 @@ import ( "github.com/formancehq/go-libs/v2/query" ) -func getPITOOTFilter(r *http.Request) (*ledgercontroller.PITFilter, error) { - pitString := r.URL.Query().Get("endTime") - ootString := r.URL.Query().Get("startTime") - - var ( - pit *time.Time - oot *time.Time - ) - - if pitString != "" { - var err error - _pit, err := time.ParseTime(pitString) - if err != nil { - return nil, err - } - - pit = &_pit - } - - if ootString != "" { - var err error - _oot, err := time.ParseTime(ootString) - if err != nil { - return nil, err - } - - oot = &_oot - } - - return &ledgercontroller.PITFilter{ - PIT: pit, - OOT: oot, - }, nil -} - -func getPITFilter(r *http.Request) (*ledgercontroller.PITFilter, error) { - pitString := r.URL.Query().Get("pit") - - var pit *time.Time - if pitString != "" { - var err error - _pit, err := time.ParseTime(pitString) - if err != nil { - return nil, err - } +func getDate(r *http.Request, key string) (*time.Time, error) { + dateString := r.URL.Query().Get(key) - pit = &_pit + if dateString == "" { + return nil, nil } - return &ledgercontroller.PITFilter{ - PIT: pit, - }, nil -} - -func getPITFilterWithVolumes(r *http.Request) (*ledgercontroller.PITFilterWithVolumes, error) { - pit, err := getPITFilter(r) + date, err := time.ParseTime(dateString) if err != nil { return nil, err } - return &ledgercontroller.PITFilterWithVolumes{ - PITFilter: *pit, - ExpandVolumes: hasExpandVolumes(r), - ExpandEffectiveVolumes: hasExpandEffectiveVolumes(r), - }, nil -} -func hasExpandVolumes(r *http.Request) bool { - parts := strings.Split(r.URL.Query().Get("expand"), ",") - return slices.Contains(parts, "volumes") + return &date, nil } -func hasExpandEffectiveVolumes(r *http.Request) bool { - parts := strings.Split(r.URL.Query().Get("expand"), ",") - return slices.Contains(parts, "effectiveVolumes") +func getPIT(r *http.Request) (*time.Time, error) { + return getDate(r, "pit") } -func getFiltersForVolumes(r *http.Request) (*ledgercontroller.FiltersForVolumes, error) { - pit, err := getPITOOTFilter(r) - if err != nil { - return nil, err - } - - useInsertionDate := api.QueryParamBool(r, "insertionDate") - groupLvl := 0 - - groupLvlStr := r.URL.Query().Get("groupBy") - if groupLvlStr != "" { - groupLvlInt, err := strconv.Atoi(groupLvlStr) - if err != nil { - return nil, err - } - if groupLvlInt > 0 { - groupLvl = groupLvlInt - } - } - return &ledgercontroller.FiltersForVolumes{ - PITFilter: *pit, - UseInsertionDate: useInsertionDate, - GroupLvl: groupLvl, - }, nil +func getOOT(r *http.Request) (*time.Time, error) { + return getDate(r, "oot") } func getQueryBuilder(r *http.Request) (query.Builder, error) { @@ -136,44 +53,80 @@ func getQueryBuilder(r *http.Request) (query.Builder, error) { return nil, nil } -func getPaginatedQueryOptionsOfPITFilterWithVolumes(r *http.Request) (*ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes], error) { - qb, err := getQueryBuilder(r) - if err != nil { - return nil, err - } +func getExpand(r *http.Request) []string { + return Flatten( + Map(r.URL.Query()["expand"], func(from string) []string { + return strings.Split(from, ",") + }), + ) +} - pitFilter, err := getPITFilterWithVolumes(r) - if err != nil { - return nil, err - } +func getOffsetPaginatedQuery[v any](r *http.Request, modifiers ...func(*v) error) (*ledgercontroller.OffsetPaginatedQuery[v], error) { + return bunpaginate.Extract[ledgercontroller.OffsetPaginatedQuery[v]](r, func() (*ledgercontroller.OffsetPaginatedQuery[v], error) { + rq, err := getResourceQuery[v](r, modifiers...) + if err != nil { + return nil, err + } - pageSize, err := bunpaginate.GetPageSize(r) - if err != nil { - return nil, err - } + pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) + if err != nil { + return nil, err + } - return pointer.For(ledgercontroller.NewPaginatedQueryOptions(*pitFilter). - WithQueryBuilder(qb). - WithPageSize(pageSize)), nil + return &ledgercontroller.OffsetPaginatedQuery[v]{ + PageSize: pageSize, + Options: *rq, + }, nil + }) } -func getPaginatedQueryOptionsOfFiltersForVolumes(r *http.Request) (*ledgercontroller.PaginatedQueryOptions[ledgercontroller.FiltersForVolumes], error) { - qb, err := getQueryBuilder(r) +func getColumnPaginatedQuery[v any](r *http.Request, defaultPaginationColumn string, order bunpaginate.Order, modifiers ...func(*v) error) (*ledgercontroller.ColumnPaginatedQuery[v], error) { + return bunpaginate.Extract[ledgercontroller.ColumnPaginatedQuery[v]](r, func() (*ledgercontroller.ColumnPaginatedQuery[v], error) { + rq, err := getResourceQuery[v](r, modifiers...) + if err != nil { + return nil, err + } + + pageSize, err := bunpaginate.GetPageSize(r, bunpaginate.WithMaxPageSize(MaxPageSize), bunpaginate.WithDefaultPageSize(DefaultPageSize)) + if err != nil { + return nil, err + } + + return &ledgercontroller.ColumnPaginatedQuery[v]{ + PageSize: pageSize, + Column: defaultPaginationColumn, + Order: pointer.For(order), + Options: *rq, + }, nil + }) +} + +func getResourceQuery[v any](r *http.Request, modifiers ...func(*v) error) (*ledgercontroller.ResourceQuery[v], error) { + pit, err := getPIT(r) if err != nil { return nil, err } - - filtersForVolumes, err := getFiltersForVolumes(r) + oot, err := getOOT(r) if err != nil { return nil, err } - - pageSize, err := bunpaginate.GetPageSize(r) + builder, err := getQueryBuilder(r) if err != nil { return nil, err } - return pointer.For(ledgercontroller.NewPaginatedQueryOptions(*filtersForVolumes). - WithPageSize(pageSize). - WithQueryBuilder(qb)), nil + var options v + for _, modifier := range modifiers { + if err := modifier(&options); err != nil { + return nil, err + } + } + + return &ledgercontroller.ResourceQuery[v]{ + PIT: pit, + OOT: oot, + Builder: builder, + Expand: getExpand(r), + Opts: options, + }, nil } diff --git a/internal/api/v2/controllers_accounts_count.go b/internal/api/v2/controllers_accounts_count.go index 61d26cd61..c8346ac08 100644 --- a/internal/api/v2/controllers_accounts_count.go +++ b/internal/api/v2/controllers_accounts_count.go @@ -13,13 +13,13 @@ import ( func countAccounts(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) + rq, err := getResourceQuery[any](r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - count, err := l.CountAccounts(r.Context(), ledgercontroller.NewListAccountsQuery(*options)) + count, err := l.CountAccounts(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_accounts_count_test.go b/internal/api/v2/controllers_accounts_count_test.go index 8e04ac775..151f66451 100644 --- a/internal/api/v2/controllers_accounts_count_test.go +++ b/internal/api/v2/controllers_accounts_count_test.go @@ -26,7 +26,7 @@ func TestAccountsCount(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + expectQuery ledgercontroller.ResourceQuery[any] expectStatusCode int expectedErrorCode string returnErr error @@ -37,82 +37,51 @@ func TestAccountsCount(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using metadata", body: `{"$match": { "metadata[roles]": "admin" }}`, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("metadata[roles]", "admin")). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("metadata[roles]", "admin"), + Expand: make([]string, 0), + }, }, { name: "using address", body: `{"$match": { "address": "foo" }}`, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("address", "foo")). - WithPageSize(DefaultPageSize), - }, - { - name: "invalid page size", - queryParams: url.Values{ - "pageSize": []string{"nan"}, + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("address", "foo"), + Expand: make([]string, 0), }, - expectStatusCode: http.StatusBadRequest, - expectedErrorCode: common.ErrValidation, - }, - { - name: "page size over maximum", - expectBackendCall: true, - queryParams: url.Values{ - "pageSize": []string{"1000000"}, - }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(MaxPageSize), }, { name: "using balance filter", body: `{"$lt": { "balance[USD/2]": 100 }}`, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Lt("balance[USD/2]", float64(100))). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Lt("balance[USD/2]", float64(100)), + Expand: make([]string, 0), + }, }, { name: "using exists filter", body: `{"$exists": { "metadata": "foo" }}`, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Exists("metadata", "foo")). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Exists("metadata", "foo"), + Expand: make([]string, 0), + }, }, { name: "using invalid query payload", @@ -126,12 +95,10 @@ func TestAccountsCount(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrInvalidQuery{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, }, { name: "with missing feature", @@ -139,12 +106,10 @@ func TestAccountsCount(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, }, { name: "with unexpected error", @@ -152,12 +117,10 @@ func TestAccountsCount(t *testing.T) { expectedErrorCode: api.ErrorInternal, expectBackendCall: true, returnErr: errors.New("undefined error"), - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithPageSize(DefaultPageSize), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, }, } for _, testCase := range testCases { @@ -171,7 +134,7 @@ func TestAccountsCount(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectBackendCall { ledgerController.EXPECT(). - CountAccounts(gomock.Any(), ledgercontroller.NewListAccountsQuery(testCase.expectQuery)). + CountAccounts(gomock.Any(), testCase.expectQuery). Return(10, testCase.returnErr) } diff --git a/internal/api/v2/controllers_accounts_list.go b/internal/api/v2/controllers_accounts_list.go index 88b3322a0..0c1e171fd 100644 --- a/internal/api/v2/controllers_accounts_list.go +++ b/internal/api/v2/controllers_accounts_list.go @@ -5,8 +5,6 @@ import ( "errors" "github.com/formancehq/go-libs/v2/api" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" - "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) @@ -14,13 +12,7 @@ import ( func listAccounts(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query, err := bunpaginate.Extract[ledgercontroller.ListAccountsQuery](r, func() (*ledgercontroller.ListAccountsQuery, error) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) - if err != nil { - return nil, err - } - return pointer.For(ledgercontroller.NewListAccountsQuery(*options)), nil - }) + query, err := getOffsetPaginatedQuery[any](r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return diff --git a/internal/api/v2/controllers_accounts_list_test.go b/internal/api/v2/controllers_accounts_list_test.go index b2c81a63b..3dff1b1de 100644 --- a/internal/api/v2/controllers_accounts_list_test.go +++ b/internal/api/v2/controllers_accounts_list_test.go @@ -29,7 +29,7 @@ func TestAccountsList(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + expectQuery ledgercontroller.OffsetPaginatedQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -40,45 +40,54 @@ func TestAccountsList(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), }, - }). - WithPageSize(DefaultPageSize), + }, expectBackendCall: true, }, { name: "using metadata", body: `{"$match": { "metadata[roles]": "admin" }}`, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("metadata[roles]", "admin"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("metadata[roles]", "admin")). - WithPageSize(DefaultPageSize), + }, }, { name: "using address", body: `{"$match": { "address": "foo" }}`, expectBackendCall: true, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("address", "foo"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("address", "foo")). - WithPageSize(DefaultPageSize), + }, }, { name: "using empty cursor", expectBackendCall: true, queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{})))}, + "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{}, + })}, + }, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}), }, { name: "using invalid cursor", @@ -102,36 +111,39 @@ func TestAccountsList(t *testing.T) { queryParams: url.Values{ "pageSize": []string{"1000000"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: MaxPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), }, - }). - WithPageSize(MaxPageSize), + }, }, { name: "using balance filter", expectBackendCall: true, body: `{"$lt": { "balance[USD/2]": 100 }}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Lt("balance[USD/2]", float64(100)), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Lt("balance[USD/2]", float64(100))). - WithPageSize(DefaultPageSize), + }, }, { name: "using exists filter", expectBackendCall: true, body: `{"$exists": { "metadata": "foo" }}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Exists("metadata", "foo"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Exists("metadata", "foo")). - WithPageSize(DefaultPageSize), + }, }, { name: "using invalid query payload", @@ -145,12 +157,13 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrInvalidQuery{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), }, - }). - WithPageSize(DefaultPageSize), + }, }, { name: "with missing feature", @@ -158,12 +171,13 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), }, - }). - WithPageSize(DefaultPageSize), + }, }, { name: "with unexpected error", @@ -171,12 +185,13 @@ func TestAccountsList(t *testing.T) { expectedErrorCode: api.ErrorInternal, expectBackendCall: true, returnErr: errors.New("undefined error"), - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), }, - }). - WithPageSize(DefaultPageSize), + }, }, } for _, testCase := range testCases { @@ -199,7 +214,7 @@ func TestAccountsList(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if tc.expectBackendCall { ledgerController.EXPECT(). - ListAccounts(gomock.Any(), ledgercontroller.NewListAccountsQuery(tc.expectQuery)). + ListAccounts(gomock.Any(), tc.expectQuery). Return(&expectedCursor, tc.returnErr) } diff --git a/internal/api/v2/controllers_accounts_read.go b/internal/api/v2/controllers_accounts_read.go index cb9f21672..e063138ad 100644 --- a/internal/api/v2/controllers_accounts_read.go +++ b/internal/api/v2/controllers_accounts_read.go @@ -1,6 +1,7 @@ package v2 import ( + "github.com/formancehq/go-libs/v2/query" "net/http" "net/url" @@ -20,21 +21,17 @@ func readAccount(w http.ResponseWriter, r *http.Request) { return } - query := ledgercontroller.NewGetAccountQuery(param) - if hasExpandVolumes(r) { - query = query.WithExpandVolumes() - } - if hasExpandEffectiveVolumes(r) { - query = query.WithExpandEffectiveVolumes() - } - pitFilter, err := getPITFilter(r) + pit, err := getPIT(r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - query.PITFilter = *pitFilter - acc, err := l.GetAccount(r.Context(), query) + acc, err := l.GetAccount(r.Context(), ledgercontroller.ResourceQuery[any]{ + PIT: pit, + Builder: query.Match("address", param), + Expand: r.URL.Query()["expand"], + }) if err != nil { switch { case postgres.IsNotFoundError(err): diff --git a/internal/api/v2/controllers_accounts_read_test.go b/internal/api/v2/controllers_accounts_read_test.go index 8ab40ccc2..259d550fd 100644 --- a/internal/api/v2/controllers_accounts_read_test.go +++ b/internal/api/v2/controllers_accounts_read_test.go @@ -2,6 +2,7 @@ package v2 import ( "bytes" + "github.com/formancehq/go-libs/v2/query" "github.com/formancehq/ledger/internal/api/common" "net/http" "net/http/httptest" @@ -25,7 +26,7 @@ func TestAccountsRead(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.GetAccountQuery + expectQuery ledgercontroller.ResourceQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -38,13 +39,20 @@ func TestAccountsRead(t *testing.T) { { name: "nominal", account: "foo", - expectQuery: ledgercontroller.NewGetAccountQuery("foo").WithPIT(before), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("address", "foo"), + }, expectBackendCall: true, }, { name: "with expand volumes", account: "foo", - expectQuery: ledgercontroller.NewGetAccountQuery("foo").WithPIT(before).WithExpandVolumes(), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("address", "foo"), + Expand: []string{"volumes"}, + }, expectBackendCall: true, queryParams: url.Values{ "expand": {"volumes"}, @@ -53,7 +61,11 @@ func TestAccountsRead(t *testing.T) { { name: "with expand effective volumes", account: "foo", - expectQuery: ledgercontroller.NewGetAccountQuery("foo").WithPIT(before).WithExpandEffectiveVolumes(), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("address", "foo"), + Expand: []string{"effectiveVolumes"}, + }, expectBackendCall: true, queryParams: url.Values{ "expand": {"effectiveVolumes"}, diff --git a/internal/api/v2/controllers_balances.go b/internal/api/v2/controllers_balances.go index a63646f38..4d094352b 100644 --- a/internal/api/v2/controllers_balances.go +++ b/internal/api/v2/controllers_balances.go @@ -12,21 +12,17 @@ import ( func readBalancesAggregated(w http.ResponseWriter, r *http.Request) { - pitFilter, err := getPITFilter(r) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } + rq, err := getResourceQuery[ledgercontroller.GetAggregatedVolumesOptions](r, func(options *ledgercontroller.GetAggregatedVolumesOptions) error { + options.UseInsertionDate = api.QueryParamBool(r, "use_insertion_date") || api.QueryParamBool(r, "useInsertionDate") - queryBuilder, err := getQueryBuilder(r) + return nil + }) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - balances, err := common.LedgerFromContext(r.Context()). - GetAggregatedBalances(r.Context(), ledgercontroller.NewGetAggregatedBalancesQuery( - *pitFilter, queryBuilder, api.QueryParamBool(r, "use_insertion_date") || api.QueryParamBool(r, "useInsertionDate"))) + balances, err := common.LedgerFromContext(r.Context()).GetAggregatedBalances(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_balances_test.go b/internal/api/v2/controllers_balances_test.go index 5d1de3a8b..1428bbd77 100644 --- a/internal/api/v2/controllers_balances_test.go +++ b/internal/api/v2/controllers_balances_test.go @@ -28,7 +28,7 @@ func TestBalancesAggregates(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.GetAggregatedBalanceQuery + expectQuery ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions] } now := time.Now() @@ -36,30 +36,30 @@ func TestBalancesAggregates(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, - }, + expectQuery: ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{}, + PIT: &now, + Expand: make([]string, 0), }, }, { name: "using address", body: `{"$match": {"address": "foo"}}`, - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, - }, - QueryBuilder: query.Match("address", "foo"), + expectQuery: ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{}, + PIT: &now, + Builder: query.Match("address", "foo"), + Expand: make([]string, 0), }, }, { name: "using exists metadata filter", body: `{"$exists": {"metadata": "foo"}}`, - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, - }, - QueryBuilder: query.Exists("metadata", "foo"), + expectQuery: ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{}, + PIT: &now, + Builder: query.Exists("metadata", "foo"), + Expand: make([]string, 0), }, }, { @@ -67,10 +67,10 @@ func TestBalancesAggregates(t *testing.T) { queryParams: url.Values{ "pit": []string{now.Format(time.RFC3339Nano)}, }, - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, - }, + expectQuery: ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{}, + PIT: &now, + Expand: make([]string, 0), }, }, { @@ -79,11 +79,12 @@ func TestBalancesAggregates(t *testing.T) { "pit": []string{now.Format(time.RFC3339Nano)}, "useInsertionDate": []string{"true"}, }, - expectQuery: ledgercontroller.GetAggregatedBalanceQuery{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{ + UseInsertionDate: true, }, - UseInsertionDate: true, + PIT: &now, + Expand: make([]string, 0), }, }, } diff --git a/internal/api/v2/controllers_logs_list.go b/internal/api/v2/controllers_logs_list.go index 231c0f278..56c82236a 100644 --- a/internal/api/v2/controllers_logs_list.go +++ b/internal/api/v2/controllers_logs_list.go @@ -1,7 +1,6 @@ package v2 import ( - "fmt" "net/http" "errors" @@ -15,36 +14,13 @@ import ( func listLogs(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query := ledgercontroller.GetLogsQuery{} - - if r.URL.Query().Get(QueryKeyCursor) != "" { - err := bunpaginate.UnmarshalCursor(r.URL.Query().Get(QueryKeyCursor), &query) - if err != nil { - api.BadRequest(w, common.ErrValidation, fmt.Errorf("invalid '%s' query param", QueryKeyCursor)) - return - } - } else { - var err error - - pageSize, err := bunpaginate.GetPageSize(r) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } - - qb, err := getQueryBuilder(r) - if err != nil { - api.BadRequest(w, common.ErrValidation, err) - return - } - - query = ledgercontroller.NewListLogsQuery(ledgercontroller.PaginatedQueryOptions[any]{ - QueryBuilder: qb, - PageSize: pageSize, - }) + rq, err := getColumnPaginatedQuery[any](r, "id", bunpaginate.OrderDesc) + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return } - cursor, err := l.ListLogs(r.Context(), query) + cursor, err := l.ListLogs(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrInvalidQuery{}): diff --git a/internal/api/v2/controllers_logs_list_test.go b/internal/api/v2/controllers_logs_list_test.go index c06db4189..2e931f7f3 100644 --- a/internal/api/v2/controllers_logs_list_test.go +++ b/internal/api/v2/controllers_logs_list_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" "net/http" "net/http/httptest" @@ -30,7 +31,7 @@ func TestGetLogs(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.PaginatedQueryOptions[any] + expectQuery ledgercontroller.ColumnPaginatedQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -40,29 +41,59 @@ func TestGetLogs(t *testing.T) { now := time.Now() testCases := []testCase{ { - name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil), + name: "nominal", + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + Expand: make([]string, 0), + }, + }, expectBackendCall: true, }, { - name: "using start time", - body: fmt.Sprintf(`{"$gte": {"date": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil).WithQueryBuilder(query.Gte("date", now.Format(time.DateFormat))), + name: "using start time", + body: fmt.Sprintf(`{"$gte": {"date": "%s"}}`, now.Format(time.DateFormat)), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Gte("date", now.Format(time.DateFormat)), + Expand: make([]string, 0), + }, + }, expectBackendCall: true, }, { name: "using end time", body: fmt.Sprintf(`{"$lt": {"date": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil). - WithQueryBuilder(query.Lt("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("date", now.Format(time.DateFormat)), + Expand: make([]string, 0), + }, + }, expectBackendCall: true, }, { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil)))}, + "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + })}, + }, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil), expectBackendCall: true, }, { @@ -88,17 +119,31 @@ func TestGetLogs(t *testing.T) { expectedErrorCode: common.ErrValidation, }, { - name: "with invalid query", - expectStatusCode: http.StatusBadRequest, - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil), + name: "with invalid query", + expectStatusCode: http.StatusBadRequest, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + Expand: make([]string, 0), + }, + }, expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrInvalidQuery{}, }, { - name: "with unexpected error", - expectStatusCode: http.StatusInternalServerError, - expectQuery: ledgercontroller.NewPaginatedQueryOptions[any](nil), + name: "with unexpected error", + expectStatusCode: http.StatusInternalServerError, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + Expand: make([]string, 0), + }, + }, expectedErrorCode: api.ErrorInternal, expectBackendCall: true, returnErr: errors.New("unexpected error"), @@ -125,7 +170,7 @@ func TestGetLogs(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectBackendCall { ledgerController.EXPECT(). - ListLogs(gomock.Any(), ledgercontroller.NewListLogsQuery(testCase.expectQuery)). + ListLogs(gomock.Any(), testCase.expectQuery). Return(&expectedCursor, testCase.returnErr) } diff --git a/internal/api/v2/controllers_transactions_count.go b/internal/api/v2/controllers_transactions_count.go index 3388d07fc..75f50bc37 100644 --- a/internal/api/v2/controllers_transactions_count.go +++ b/internal/api/v2/controllers_transactions_count.go @@ -12,14 +12,13 @@ import ( func countTransactions(w http.ResponseWriter, r *http.Request) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) + rq, err := getResourceQuery[any](r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - count, err := common.LedgerFromContext(r.Context()). - CountTransactions(r.Context(), ledgercontroller.NewListTransactionsQuery(*options)) + count, err := common.LedgerFromContext(r.Context()).CountTransactions(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_transactions_count_test.go b/internal/api/v2/controllers_transactions_count_test.go index a7a53a4a0..7cd83cd7e 100644 --- a/internal/api/v2/controllers_transactions_count_test.go +++ b/internal/api/v2/controllers_transactions_count_test.go @@ -29,7 +29,7 @@ func TestTransactionsCount(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + expectQuery ledgercontroller.ResourceQuery[any] expectStatusCode int expectedErrorCode string expectBackendCall bool @@ -40,97 +40,88 @@ func TestTransactionsCount(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using metadata", body: `{"$match": {"metadata[roles]": "admin"}}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("metadata[roles]", "admin")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("metadata[roles]", "admin"), + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using startTime", body: fmt.Sprintf(`{"$gte": {"date": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Gte("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Gte("date", now.Format(time.DateFormat)), + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using endTime", body: fmt.Sprintf(`{"$gte": {"date": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Gte("date", now.Format(time.DateFormat))), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Gte("date", now.Format(time.DateFormat)), + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using account", body: `{"$match": {"account": "xxx"}}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("account", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("account", "xxx"), + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using reference", body: `{"$match": {"reference": "xxx"}}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("reference", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("reference", "xxx"), + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using destination", body: `{"$match": {"destination": "xxx"}}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("destination", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("destination", "xxx"), + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "using source", body: `{"$match": {"source": "xxx"}}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }). - WithQueryBuilder(query.Match("source", "xxx")), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Builder: query.Match("source", "xxx"), + Expand: make([]string, 0), + }, expectBackendCall: true, }, { name: "error from backend", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, expectStatusCode: http.StatusInternalServerError, expectedErrorCode: api.ErrorInternal, expectBackendCall: true, @@ -142,11 +133,10 @@ func TestTransactionsCount(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrInvalidQuery{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, }, { name: "with missing feature", @@ -154,11 +144,10 @@ func TestTransactionsCount(t *testing.T) { expectedErrorCode: common.ErrValidation, expectBackendCall: true, returnErr: ledgercontroller.ErrMissingFeature{}, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, - }, - }), + expectQuery: ledgercontroller.ResourceQuery[any]{ + PIT: &before, + Expand: make([]string, 0), + }, }, } for _, tc := range testCases { @@ -171,7 +160,7 @@ func TestTransactionsCount(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if tc.expectBackendCall { ledgerController.EXPECT(). - CountTransactions(gomock.Any(), ledgercontroller.NewListTransactionsQuery(tc.expectQuery)). + CountTransactions(gomock.Any(), tc.expectQuery). Return(10, tc.returnErr) } diff --git a/internal/api/v2/controllers_transactions_list.go b/internal/api/v2/controllers_transactions_list.go index b64839dac..2705a514e 100644 --- a/internal/api/v2/controllers_transactions_list.go +++ b/internal/api/v2/controllers_transactions_list.go @@ -6,7 +6,6 @@ import ( "errors" "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/go-libs/v2/bun/bunpaginate" - "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) @@ -14,28 +13,18 @@ import ( func listTransactions(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query, err := bunpaginate.Extract[ledgercontroller.ListTransactionsQuery](r, func() (*ledgercontroller.ListTransactionsQuery, error) { - options, err := getPaginatedQueryOptionsOfPITFilterWithVolumes(r) - if err != nil { - return nil, err - } - q := ledgercontroller.NewListTransactionsQuery(*options) - - if r.URL.Query().Get("order") == "effective" { - q.Column = "timestamp" - } - if r.URL.Query().Get("reverse") == "true" { - q.Order = bunpaginate.OrderAsc - } + paginationColumn := "id" + if r.URL.Query().Get("order") == "effective" { + paginationColumn = "timestamp" + } - return pointer.For(q), nil - }) + rq, err := getColumnPaginatedQuery[any](r, paginationColumn, bunpaginate.OrderDesc) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - cursor, err := l.ListTransactions(r.Context(), *query) + cursor, err := l.ListTransactions(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_transactions_list_test.go b/internal/api/v2/controllers_transactions_list_test.go index 8ff17eff3..c64489bb7 100644 --- a/internal/api/v2/controllers_transactions_list_test.go +++ b/internal/api/v2/controllers_transactions_list_test.go @@ -3,6 +3,7 @@ package v2 import ( "bytes" "fmt" + "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/ledger/internal/api/common" "math/big" "net/http" @@ -29,7 +30,7 @@ func TestTransactionsList(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.ListTransactionsQuery + expectQuery ledgercontroller.ColumnPaginatedQuery[any] expectStatusCode int expectedErrorCode string } @@ -38,90 +39,120 @@ func TestTransactionsList(t *testing.T) { testCases := []testCase{ { name: "nominal", - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Expand: make([]string, 0), }, - })), + }, }, { name: "using metadata", body: `{"$match": {"metadata[roles]": "admin"}}`, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Match("metadata[roles]", "admin"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("metadata[roles]", "admin"))), + }, }, { name: "using startTime", body: fmt.Sprintf(`{"$gte": {"start_time": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Gte("start_time", now.Format(time.DateFormat)), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Gte("start_time", now.Format(time.DateFormat)))), + }, }, { name: "using endTime", body: fmt.Sprintf(`{"$lte": {"end_time": "%s"}}`, now.Format(time.DateFormat)), - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Lte("end_time", now.Format(time.DateFormat)), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Lte("end_time", now.Format(time.DateFormat)))), + }, }, { name: "using account", body: `{"$match": {"account": "xxx"}}`, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Match("account", "xxx"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("account", "xxx"))), + }, }, { name: "using reference", body: `{"$match": {"reference": "xxx"}}`, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Match("reference", "xxx"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("reference", "xxx"))), + }, }, { name: "using destination", body: `{"$match": {"destination": "xxx"}}`, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Expand: make([]string, 0), + Builder: query.Match("destination", "xxx"), }, - }). - WithQueryBuilder(query.Match("destination", "xxx"))), + }, }, { name: "using source", body: `{"$match": {"source": "xxx"}}`, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Match("source", "xxx"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("source", "xxx"))), + }, }, { name: "using empty cursor", queryParams: url.Values{ - "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{})))}, + "cursor": []string{bunpaginate.EncodeCursor(ledgercontroller.ColumnPaginatedQuery[any]{})}, }, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{}, - })), + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{}, }, { name: "using invalid cursor", @@ -144,39 +175,65 @@ func TestTransactionsList(t *testing.T) { queryParams: url.Values{ "pageSize": []string{"1000000"}, }, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: MaxPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Expand: make([]string, 0), }, - }). - WithPageSize(MaxPageSize)), + }, }, { name: "using cursor", queryParams: url.Values{ - "cursor": []string{"eyJwYWdlU2l6ZSI6MTUsImJvdHRvbSI6bnVsbCwiY29sdW1uIjoiaWQiLCJwYWdpbmF0aW9uSUQiOm51bGwsIm9yZGVyIjoxLCJmaWx0ZXJzIjp7InFiIjp7fSwicGFnZVNpemUiOjE1LCJvcHRpb25zIjp7InBpdCI6bnVsbCwidm9sdW1lcyI6ZmFsc2UsImVmZmVjdGl2ZVZvbHVtZXMiOmZhbHNlfX0sInJldmVyc2UiOmZhbHNlfQ"}, + "cursor": []string{func() string { + return bunpaginate.EncodeCursor(ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + }, + }) + }()}, + }, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + }, }, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{})), }, { name: "using $exists metadata filter", body: `{"$exists": {"metadata": "foo"}}`, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "id", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Exists("metadata", "foo"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Exists("metadata", "foo"))), + }, }, { name: "paginate using effective order", queryParams: map[string][]string{"order": {"effective"}}, - expectQuery: ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + expectQuery: ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: DefaultPageSize, + Column: "timestamp", + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Options: ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Expand: make([]string, 0), }, - })). - WithColumn("timestamp"), + }, }, } for _, testCase := range testCases { @@ -223,7 +280,6 @@ func TestTransactionsList(t *testing.T) { err := api.ErrorResponse{} api.Decode(t, rec.Body, &err) require.EqualValues(t, testCase.expectedErrorCode, err.ErrorCode) - } }) } diff --git a/internal/api/v2/controllers_transactions_read.go b/internal/api/v2/controllers_transactions_read.go index ef00f5e00..b22142cd6 100644 --- a/internal/api/v2/controllers_transactions_read.go +++ b/internal/api/v2/controllers_transactions_read.go @@ -1,6 +1,7 @@ package v2 import ( + "github.com/formancehq/go-libs/v2/query" "net/http" "strconv" @@ -20,22 +21,17 @@ func readTransaction(w http.ResponseWriter, r *http.Request) { return } - query := ledgercontroller.NewGetTransactionQuery(int(txId)) - if hasExpandVolumes(r) { - query = query.WithExpandVolumes() - } - if hasExpandEffectiveVolumes(r) { - query = query.WithExpandEffectiveVolumes() - } - - pitFilter, err := getPITFilter(r) + pit, err := getPIT(r) if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - query.PITFilter = *pitFilter - tx, err := l.GetTransaction(r.Context(), query) + tx, err := l.GetTransaction(r.Context(), ledgercontroller.ResourceQuery[any]{ + PIT: pit, + Builder: query.Match("id", int(txId)), + Expand: r.URL.Query()["expand"], + }) if err != nil { switch { case postgres.IsNotFoundError(err): diff --git a/internal/api/v2/controllers_transactions_read_test.go b/internal/api/v2/controllers_transactions_read_test.go index 8eba1283f..4033ca7d9 100644 --- a/internal/api/v2/controllers_transactions_read_test.go +++ b/internal/api/v2/controllers_transactions_read_test.go @@ -1,6 +1,7 @@ package v2 import ( + "github.com/formancehq/go-libs/v2/query" "math/big" "net/http" "net/http/httptest" @@ -25,12 +26,15 @@ func TestTransactionsRead(t *testing.T) { ledger.NewPosting("world", "bank", "USD", big.NewInt(100)), ) - query := ledgercontroller.NewGetTransactionQuery(0) - query.PIT = &now + q := ledgercontroller.ResourceQuery[any]{ + PIT: &now, + Builder: query.Match("id", tx.ID), + } + q.PIT = &now systemController, ledgerController := newTestingSystemController(t, true) ledgerController.EXPECT(). - GetTransaction(gomock.Any(), query). + GetTransaction(gomock.Any(), q). Return(&tx, nil) router := NewRouter(systemController, auth.NewNoAuth(), os.Getenv("DEBUG") == "true") diff --git a/internal/api/v2/controllers_volumes.go b/internal/api/v2/controllers_volumes.go index caef27739..8c94765cb 100644 --- a/internal/api/v2/controllers_volumes.go +++ b/internal/api/v2/controllers_volumes.go @@ -2,40 +2,55 @@ package v2 import ( "net/http" + "strconv" "errors" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/go-libs/v2/api" "github.com/formancehq/ledger/internal/api/common" - - "github.com/formancehq/go-libs/v2/pointer" - - "github.com/formancehq/go-libs/v2/bun/bunpaginate" ) func readVolumes(w http.ResponseWriter, r *http.Request) { l := common.LedgerFromContext(r.Context()) - query, err := bunpaginate.Extract[ledgercontroller.GetVolumesWithBalancesQuery](r, func() (*ledgercontroller.GetVolumesWithBalancesQuery, error) { - options, err := getPaginatedQueryOptionsOfFiltersForVolumes(r) - if err != nil { - return nil, err + rq, err := getOffsetPaginatedQuery[ledgercontroller.GetVolumesOptions](r, func(opts *ledgercontroller.GetVolumesOptions) error { + groupBy := r.URL.Query().Get("groupBy") + if groupBy != "" { + v, err := strconv.ParseInt(groupBy, 10, 64) + if err != nil { + return err + } + opts.GroupLvl = int(v) } - getVolumesWithBalancesQuery := ledgercontroller.NewGetVolumesWithBalancesQuery(*options) - return pointer.For(getVolumesWithBalancesQuery), nil + opts.UseInsertionDate = api.QueryParamBool(r, "insertionDate") + return nil }) - if err != nil { api.BadRequest(w, common.ErrValidation, err) return } - cursor, err := l.GetVolumesWithBalances(r.Context(), *query) + if r.URL.Query().Get("endTime") != "" { + rq.Options.PIT, err = getDate(r, "endTime") + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } + } + + if r.URL.Query().Get("startTime") != "" { + rq.Options.OOT, err = getDate(r, "startTime") + if err != nil { + api.BadRequest(w, common.ErrValidation, err) + return + } + } + cursor, err := l.GetVolumesWithBalances(r.Context(), *rq) if err != nil { switch { case errors.Is(err, ledgercontroller.ErrInvalidQuery{}) || errors.Is(err, ledgercontroller.ErrMissingFeature{}): diff --git a/internal/api/v2/controllers_volumes_test.go b/internal/api/v2/controllers_volumes_test.go index 7d5b7a183..b570bae9b 100644 --- a/internal/api/v2/controllers_volumes_test.go +++ b/internal/api/v2/controllers_volumes_test.go @@ -31,7 +31,7 @@ func TestGetVolumes(t *testing.T) { name string queryParams url.Values body string - expectQuery ledgercontroller.PaginatedQueryOptions[ledgercontroller.FiltersForVolumes] + expectQuery ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions] expectStatusCode int expectedErrorCode string } @@ -40,36 +40,37 @@ func TestGetVolumes(t *testing.T) { testCases := []testCase{ { name: "basic", - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &before, + Expand: make([]string, 0), }, - - UseInsertionDate: false, - }). - WithPageSize(DefaultPageSize), + }, }, { name: "using metadata", body: `{"$match": { "metadata[roles]": "admin" }}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &before, + Builder: query.Match("metadata[roles]", "admin"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("metadata[roles]", "admin")). - WithPageSize(DefaultPageSize), + }, }, { name: "using account", body: `{"$match": { "account": "foo" }}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &before, + Builder: query.Match("account", "foo"), + Expand: make([]string, 0), }, - }). - WithQueryBuilder(query.Match("account", "foo")). - WithPageSize(DefaultPageSize), + }, }, { name: "using invalid query payload", @@ -83,31 +84,40 @@ func TestGetVolumes(t *testing.T) { "pit": []string{before.Format(time.RFC3339Nano)}, "groupBy": []string{"3"}, }, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &before, + Expand: make([]string, 0), + Opts: ledgercontroller.GetVolumesOptions{ + GroupLvl: 3, + }, }, - GroupLvl: 3, - }).WithPageSize(DefaultPageSize), + }, }, { name: "using Exists metadata filter", body: `{"$exists": { "metadata": "foo" }}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &before, + Builder: query.Exists("metadata", "foo"), + Expand: make([]string, 0), }, - }).WithPageSize(DefaultPageSize).WithQueryBuilder(query.Exists("metadata", "foo")), + }, }, { name: "using balance filter", body: `{"$gte": { "balance[EUR]": 50 }}`, - expectQuery: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &before, + expectQuery: ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + PageSize: DefaultPageSize, + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &before, + Builder: query.Gte("balance[EUR]", float64(50)), + Expand: make([]string, 0), }, - }).WithQueryBuilder(query.Gte("balance[EUR]", float64(50))). - WithPageSize(DefaultPageSize), + }, }, } @@ -136,7 +146,7 @@ func TestGetVolumes(t *testing.T) { systemController, ledgerController := newTestingSystemController(t, true) if testCase.expectStatusCode < 300 && testCase.expectStatusCode >= 200 { ledgerController.EXPECT(). - GetVolumesWithBalances(gomock.Any(), ledgercontroller.NewGetVolumesWithBalancesQuery(testCase.expectQuery)). + GetVolumesWithBalances(gomock.Any(), testCase.expectQuery). Return(&expectedCursor, nil) } diff --git a/internal/api/v2/mocks_ledger_controller_test.go b/internal/api/v2/mocks_ledger_controller_test.go index 26775c2d7..2cbbfee4a 100644 --- a/internal/api/v2/mocks_ledger_controller_test.go +++ b/internal/api/v2/mocks_ledger_controller_test.go @@ -70,7 +70,7 @@ func (mr *LedgerControllerMockRecorder) Commit(ctx any) *gomock.Call { } // CountAccounts mocks base method. -func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (int, error) { +func (m *LedgerController) CountAccounts(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountAccounts", ctx, query) ret0, _ := ret[0].(int) @@ -85,7 +85,7 @@ func (mr *LedgerControllerMockRecorder) CountAccounts(ctx, query any) *gomock.Ca } // CountTransactions mocks base method. -func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (int, error) { +func (m *LedgerController) CountTransactions(ctx context.Context, query ledger0.ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountTransactions", ctx, query) ret0, _ := ret[0].(int) @@ -160,7 +160,7 @@ func (mr *LedgerControllerMockRecorder) Export(ctx, w any) *gomock.Call { } // GetAccount mocks base method. -func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.GetAccountQuery) (*ledger.Account, error) { +func (m *LedgerController) GetAccount(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Account, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAccount", ctx, query) ret0, _ := ret[0].(*ledger.Account) @@ -175,7 +175,7 @@ func (mr *LedgerControllerMockRecorder) GetAccount(ctx, query any) *gomock.Call } // GetAggregatedBalances mocks base method. -func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { +func (m *LedgerController) GetAggregatedBalances(ctx context.Context, q ledger0.ResourceQuery[ledger0.GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAggregatedBalances", ctx, q) ret0, _ := ret[0].(ledger.BalancesByAssets) @@ -220,7 +220,7 @@ func (mr *LedgerControllerMockRecorder) GetStats(ctx any) *gomock.Call { } // GetTransaction mocks base method. -func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.GetTransactionQuery) (*ledger.Transaction, error) { +func (m *LedgerController) GetTransaction(ctx context.Context, query ledger0.ResourceQuery[any]) (*ledger.Transaction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTransaction", ctx, query) ret0, _ := ret[0].(*ledger.Transaction) @@ -235,7 +235,7 @@ func (mr *LedgerControllerMockRecorder) GetTransaction(ctx, query any) *gomock.C } // GetVolumesWithBalances mocks base method. -func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *LedgerController) GetVolumesWithBalances(ctx context.Context, q ledger0.OffsetPaginatedQuery[ledger0.GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -279,7 +279,7 @@ func (mr *LedgerControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call } // ListAccounts mocks base method. -func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *LedgerController) ListAccounts(ctx context.Context, query ledger0.OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -294,7 +294,7 @@ func (mr *LedgerControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Cal } // ListLogs mocks base method. -func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *LedgerController) ListLogs(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -309,7 +309,7 @@ func (mr *LedgerControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *LedgerController) ListTransactions(ctx context.Context, query ledger0.ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/controller/ledger/controller.go b/internal/controller/ledger/controller.go index eee5771a8..a1cd360ed 100644 --- a/internal/controller/ledger/controller.go +++ b/internal/controller/ledger/controller.go @@ -24,15 +24,15 @@ type Controller interface { GetMigrationsInfo(ctx context.Context) ([]migrations.Info, error) GetStats(ctx context.Context) (Stats, error) - GetAccount(ctx context.Context, query GetAccountQuery) (*ledger.Account, error) - ListAccounts(ctx context.Context, query ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) - CountAccounts(ctx context.Context, query ListAccountsQuery) (int, error) - ListLogs(ctx context.Context, query GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) - CountTransactions(ctx context.Context, query ListTransactionsQuery) (int, error) - ListTransactions(ctx context.Context, query ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) - GetTransaction(ctx context.Context, query GetTransactionQuery) (*ledger.Transaction, error) - GetVolumesWithBalances(ctx context.Context, q GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) - GetAggregatedBalances(ctx context.Context, q GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) + GetAccount(ctx context.Context, query ResourceQuery[any]) (*ledger.Account, error) + ListAccounts(ctx context.Context, query OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) + CountAccounts(ctx context.Context, query ResourceQuery[any]) (int, error) + ListLogs(ctx context.Context, query ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) + CountTransactions(ctx context.Context, query ResourceQuery[any]) (int, error) + ListTransactions(ctx context.Context, query ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) + GetTransaction(ctx context.Context, query ResourceQuery[any]) (*ledger.Transaction, error) + GetVolumesWithBalances(ctx context.Context, q OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) + GetAggregatedBalances(ctx context.Context, q ResourceQuery[GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) // CreateTransaction accept a numscript script and returns a transaction // It can return following errors: diff --git a/internal/controller/ledger/controller_default.go b/internal/controller/ledger/controller_default.go index a8a3f26ef..eac5a7f9e 100644 --- a/internal/controller/ledger/controller_default.go +++ b/internal/controller/ledger/controller_default.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/go-libs/v2/time" "github.com/formancehq/ledger/pkg/features" "math/big" @@ -106,40 +107,52 @@ func NewDefaultController( return ret } +func (ctrl *DefaultController) IsDatabaseUpToDate(ctx context.Context) (bool, error) { + return ctrl.store.IsUpToDate(ctx) +} + func (ctrl *DefaultController) GetMigrationsInfo(ctx context.Context) ([]migrations.Info, error) { return ctrl.store.GetMigrationsInfo(ctx) } -func (ctrl *DefaultController) ListTransactions(ctx context.Context, q ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { - return ctrl.store.ListTransactions(ctx, q) +func (ctrl *DefaultController) ListTransactions(ctx context.Context, q ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { + return ctrl.store.Transactions().Paginate(ctx, q) } -func (ctrl *DefaultController) CountTransactions(ctx context.Context, q ListTransactionsQuery) (int, error) { - return ctrl.store.CountTransactions(ctx, q) +func (ctrl *DefaultController) CountTransactions(ctx context.Context, q ResourceQuery[any]) (int, error) { + return ctrl.store.Transactions().Count(ctx, q) } -func (ctrl *DefaultController) GetTransaction(ctx context.Context, query GetTransactionQuery) (*ledger.Transaction, error) { - return ctrl.store.GetTransaction(ctx, query) +func (ctrl *DefaultController) GetTransaction(ctx context.Context, q ResourceQuery[any]) (*ledger.Transaction, error) { + return ctrl.store.Transactions().GetOne(ctx, q) } -func (ctrl *DefaultController) CountAccounts(ctx context.Context, a ListAccountsQuery) (int, error) { - return ctrl.store.CountAccounts(ctx, a) +func (ctrl *DefaultController) CountAccounts(ctx context.Context, q ResourceQuery[any]) (int, error) { + return ctrl.store.Accounts().Count(ctx, q) } -func (ctrl *DefaultController) ListAccounts(ctx context.Context, a ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { - return ctrl.store.ListAccounts(ctx, a) +func (ctrl *DefaultController) ListAccounts(ctx context.Context, q OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { + return ctrl.store.Accounts().Paginate(ctx, q) } -func (ctrl *DefaultController) GetAccount(ctx context.Context, q GetAccountQuery) (*ledger.Account, error) { - return ctrl.store.GetAccount(ctx, q) +func (ctrl *DefaultController) GetAccount(ctx context.Context, q ResourceQuery[any]) (*ledger.Account, error) { + return ctrl.store.Accounts().GetOne(ctx, q) +} + +func (ctrl *DefaultController) GetAggregatedBalances(ctx context.Context, q ResourceQuery[GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { + ret, err := ctrl.store.AggregatedBalances().GetOne(ctx, q) + if err != nil { + return nil, err + } + return ret.Aggregated.Balances(), nil } -func (ctrl *DefaultController) GetAggregatedBalances(ctx context.Context, q GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { - return ctrl.store.GetAggregatedBalances(ctx, q) +func (ctrl *DefaultController) ListLogs(ctx context.Context, q ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { + return ctrl.store.Logs().Paginate(ctx, q) } -func (ctrl *DefaultController) ListLogs(ctx context.Context, q GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { - return ctrl.store.ListLogs(ctx, q) +func (ctrl *DefaultController) GetVolumesWithBalances(ctx context.Context, q OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { + return ctrl.store.Volumes().Paginate(ctx, q) } func (ctrl *DefaultController) Import(ctx context.Context, stream chan ledger.Log) error { @@ -174,9 +187,9 @@ func (ctrl *DefaultController) importLogs(ctx context.Context, store Store, stre } // We can import only if the ledger is empty. - logs, err := store.ListLogs(ctx, NewListLogsQuery(PaginatedQueryOptions[any]{ + logs, err := store.Logs().Paginate(ctx, ColumnPaginatedQuery[any]{ PageSize: 1, - })) + }) if err != nil { return fmt.Errorf("error listing logs: %w", err) } @@ -276,10 +289,12 @@ func (ctrl *DefaultController) importLog(ctx context.Context, store Store, log l func (ctrl *DefaultController) Export(ctx context.Context, w ExportWriter) error { return bunpaginate.Iterate( ctx, - NewListLogsQuery(NewPaginatedQueryOptions[any](nil).WithPageSize(100)). - WithOrder(bunpaginate.OrderAsc), - func(ctx context.Context, q GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { - return ctrl.store.ListLogs(ctx, q) + ColumnPaginatedQuery[any]{ + PageSize: 100, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }, + func(ctx context.Context, q ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { + return ctrl.store.Logs().Paginate(ctx, q) }, func(cursor *bunpaginate.Cursor[ledger.Log]) error { for _, data := range cursor.Data { @@ -292,14 +307,6 @@ func (ctrl *DefaultController) Export(ctx context.Context, w ExportWriter) error ) } -func (ctrl *DefaultController) IsDatabaseUpToDate(ctx context.Context) (bool, error) { - return ctrl.store.IsUpToDate(ctx) -} - -func (ctrl *DefaultController) GetVolumesWithBalances(ctx context.Context, q GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { - return ctrl.store.GetVolumesWithBalances(ctx, q) -} - func (ctrl *DefaultController) createTransaction(ctx context.Context, store Store, parameters Parameters[RunScript]) (*ledger.CreatedTransaction, error) { logger := logging.FromContext(ctx).WithField("req", uuid.NewString()[:8]) diff --git a/internal/controller/ledger/controller_default_test.go b/internal/controller/ledger/controller_default_test.go index d590b19c8..775414c8d 100644 --- a/internal/controller/ledger/controller_default_test.go +++ b/internal/controller/ledger/controller_default_test.go @@ -2,6 +2,7 @@ package ledger import ( "context" + "github.com/formancehq/go-libs/v2/query" "math/big" "testing" @@ -199,15 +200,24 @@ func TestListTransactions(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + transactions := NewMockPaginatedResource[ledger.Transaction, any, ColumnPaginatedQuery[any]](ctrl) cursor := &bunpaginate.Cursor[ledger.Transaction]{} - query := NewListTransactionsQuery(NewPaginatedQueryOptions[PITFilterWithVolumes](PITFilterWithVolumes{})) - store.EXPECT(). - ListTransactions(gomock.Any(), query). + store.EXPECT().Transactions().Return(transactions) + transactions.EXPECT(). + Paginate(gomock.Any(), ColumnPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + }). Return(cursor, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - ret, err := l.ListTransactions(ctx, query) + ret, err := l.ListTransactions(ctx, ColumnPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + }) require.NoError(t, err) require.Equal(t, cursor, ret) } @@ -219,12 +229,13 @@ func TestCountAccounts(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + accounts := NewMockPaginatedResource[ledger.Account, any, OffsetPaginatedQuery[any]](ctrl) - query := NewListAccountsQuery(NewPaginatedQueryOptions[PITFilterWithVolumes](PITFilterWithVolumes{})) - store.EXPECT().CountAccounts(gomock.Any(), query).Return(1, nil) + store.EXPECT().Accounts().Return(accounts) + accounts.EXPECT().Count(gomock.Any(), ResourceQuery[any]{}).Return(1, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - count, err := l.CountAccounts(ctx, query) + count, err := l.CountAccounts(ctx, ResourceQuery[any]{}) require.NoError(t, err) require.Equal(t, 1, count) } @@ -236,15 +247,18 @@ func TestGetTransaction(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + transactions := NewMockPaginatedResource[ledger.Transaction, any, ColumnPaginatedQuery[any]](ctrl) tx := ledger.Transaction{} - query := NewGetTransactionQuery(0) - store.EXPECT(). - GetTransaction(gomock.Any(), query). - Return(&tx, nil) + store.EXPECT().Transactions().Return(transactions) + transactions.EXPECT().GetOne(gomock.Any(), ResourceQuery[any]{ + Builder: query.Match("id", 1), + }).Return(&tx, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - ret, err := l.GetTransaction(ctx, query) + ret, err := l.GetTransaction(ctx, ResourceQuery[any]{ + Builder: query.Match("id", 1), + }) require.NoError(t, err) require.Equal(t, tx, *ret) } @@ -256,15 +270,18 @@ func TestGetAccount(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + accounts := NewMockPaginatedResource[ledger.Account, any, OffsetPaginatedQuery[any]](ctrl) account := ledger.Account{} - query := NewGetAccountQuery("world") - store.EXPECT(). - GetAccount(gomock.Any(), query). - Return(&account, nil) + store.EXPECT().Accounts().Return(accounts) + accounts.EXPECT().GetOne(gomock.Any(), ResourceQuery[any]{ + Builder: query.Match("address", "world"), + }).Return(&account, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - ret, err := l.GetAccount(ctx, query) + ret, err := l.GetAccount(ctx, ResourceQuery[any]{ + Builder: query.Match("address", "world"), + }) require.NoError(t, err) require.Equal(t, account, *ret) } @@ -276,12 +293,13 @@ func TestCountTransactions(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + transactions := NewMockPaginatedResource[ledger.Transaction, any, ColumnPaginatedQuery[any]](ctrl) - query := NewListTransactionsQuery(NewPaginatedQueryOptions[PITFilterWithVolumes](PITFilterWithVolumes{})) - store.EXPECT().CountTransactions(gomock.Any(), query).Return(1, nil) + store.EXPECT().Transactions().Return(transactions) + transactions.EXPECT().Count(gomock.Any(), ResourceQuery[any]{}).Return(1, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - count, err := l.CountTransactions(ctx, query) + count, err := l.CountTransactions(ctx, ResourceQuery[any]{}) require.NoError(t, err) require.Equal(t, 1, count) } @@ -293,15 +311,20 @@ func TestListAccounts(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + accounts := NewMockPaginatedResource[ledger.Account, any, OffsetPaginatedQuery[any]](ctrl) cursor := &bunpaginate.Cursor[ledger.Account]{} - query := NewListAccountsQuery(NewPaginatedQueryOptions[PITFilterWithVolumes](PITFilterWithVolumes{})) - store.EXPECT(). - ListAccounts(gomock.Any(), query). - Return(cursor, nil) + store.EXPECT().Accounts().Return(accounts) + accounts.EXPECT().Paginate(gomock.Any(), OffsetPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }).Return(cursor, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - ret, err := l.ListAccounts(ctx, query) + ret, err := l.ListAccounts(ctx, OffsetPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }) require.NoError(t, err) require.Equal(t, cursor, ret) } @@ -313,17 +336,16 @@ func TestGetAggregatedBalances(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + aggregatedBalances := NewMockResource[ledger.AggregatedVolumes, GetAggregatedVolumesOptions](ctrl) - balancesByAssets := ledger.BalancesByAssets{} - query := NewGetAggregatedBalancesQuery(PITFilter{}, nil, false) - store.EXPECT(). - GetAggregatedBalances(gomock.Any(), query). - Return(balancesByAssets, nil) + store.EXPECT().AggregatedBalances().Return(aggregatedBalances) + aggregatedBalances.EXPECT().GetOne(gomock.Any(), ResourceQuery[GetAggregatedVolumesOptions]{}). + Return(&ledger.AggregatedVolumes{}, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - ret, err := l.GetAggregatedBalances(ctx, query) + ret, err := l.GetAggregatedBalances(ctx, ResourceQuery[GetAggregatedVolumesOptions]{}) require.NoError(t, err) - require.Equal(t, balancesByAssets, ret) + require.Equal(t, ledger.BalancesByAssets{}, ret) } func TestListLogs(t *testing.T) { @@ -333,15 +355,22 @@ func TestListLogs(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + logs := NewMockPaginatedResource[ledger.Log, any, ColumnPaginatedQuery[any]](ctrl) cursor := &bunpaginate.Cursor[ledger.Log]{} - query := NewListLogsQuery(NewPaginatedQueryOptions[any](nil)) - store.EXPECT(). - ListLogs(gomock.Any(), query). - Return(cursor, nil) + store.EXPECT().Logs().Return(logs) + logs.EXPECT().Paginate(gomock.Any(), ColumnPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + }).Return(cursor, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - ret, err := l.ListLogs(ctx, query) + ret, err := l.ListLogs(ctx, ColumnPaginatedQuery[any]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderDesc)), + Column: "id", + }) require.NoError(t, err) require.Equal(t, cursor, ret) } @@ -353,15 +382,20 @@ func TestGetVolumesWithBalances(t *testing.T) { store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) ctx := logging.TestingContext() + volumes := NewMockPaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, OffsetPaginatedQuery[GetVolumesOptions]](ctrl) balancesByAssets := &bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]{} - query := NewGetVolumesWithBalancesQuery(NewPaginatedQueryOptions[FiltersForVolumes](FiltersForVolumes{})) - store.EXPECT(). - GetVolumesWithBalances(gomock.Any(), query). - Return(balancesByAssets, nil) + store.EXPECT().Volumes().Return(volumes) + volumes.EXPECT().Paginate(gomock.Any(), OffsetPaginatedQuery[GetVolumesOptions]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }).Return(balancesByAssets, nil) l := NewDefaultController(ledger.Ledger{}, store, parser) - ret, err := l.GetVolumesWithBalances(ctx, query) + ret, err := l.GetVolumesWithBalances(ctx, OffsetPaginatedQuery[GetVolumesOptions]{ + PageSize: bunpaginate.QueryDefaultPageSize, + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }) require.NoError(t, err) require.Equal(t, balancesByAssets, ret) } diff --git a/internal/controller/ledger/controller_generated_test.go b/internal/controller/ledger/controller_generated_test.go index 3090e4922..7e6601231 100644 --- a/internal/controller/ledger/controller_generated_test.go +++ b/internal/controller/ledger/controller_generated_test.go @@ -69,7 +69,7 @@ func (mr *MockControllerMockRecorder) Commit(ctx any) *gomock.Call { } // CountAccounts mocks base method. -func (m *MockController) CountAccounts(ctx context.Context, query ListAccountsQuery) (int, error) { +func (m *MockController) CountAccounts(ctx context.Context, query ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountAccounts", ctx, query) ret0, _ := ret[0].(int) @@ -84,7 +84,7 @@ func (mr *MockControllerMockRecorder) CountAccounts(ctx, query any) *gomock.Call } // CountTransactions mocks base method. -func (m *MockController) CountTransactions(ctx context.Context, query ListTransactionsQuery) (int, error) { +func (m *MockController) CountTransactions(ctx context.Context, query ResourceQuery[any]) (int, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CountTransactions", ctx, query) ret0, _ := ret[0].(int) @@ -159,7 +159,7 @@ func (mr *MockControllerMockRecorder) Export(ctx, w any) *gomock.Call { } // GetAccount mocks base method. -func (m *MockController) GetAccount(ctx context.Context, query GetAccountQuery) (*ledger.Account, error) { +func (m *MockController) GetAccount(ctx context.Context, query ResourceQuery[any]) (*ledger.Account, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAccount", ctx, query) ret0, _ := ret[0].(*ledger.Account) @@ -174,7 +174,7 @@ func (mr *MockControllerMockRecorder) GetAccount(ctx, query any) *gomock.Call { } // GetAggregatedBalances mocks base method. -func (m *MockController) GetAggregatedBalances(ctx context.Context, q GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { +func (m *MockController) GetAggregatedBalances(ctx context.Context, q ResourceQuery[GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAggregatedBalances", ctx, q) ret0, _ := ret[0].(ledger.BalancesByAssets) @@ -219,7 +219,7 @@ func (mr *MockControllerMockRecorder) GetStats(ctx any) *gomock.Call { } // GetTransaction mocks base method. -func (m *MockController) GetTransaction(ctx context.Context, query GetTransactionQuery) (*ledger.Transaction, error) { +func (m *MockController) GetTransaction(ctx context.Context, query ResourceQuery[any]) (*ledger.Transaction, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetTransaction", ctx, query) ret0, _ := ret[0].(*ledger.Transaction) @@ -234,7 +234,7 @@ func (mr *MockControllerMockRecorder) GetTransaction(ctx, query any) *gomock.Cal } // GetVolumesWithBalances mocks base method. -func (m *MockController) GetVolumesWithBalances(ctx context.Context, q GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (m *MockController) GetVolumesWithBalances(ctx context.Context, q OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) @@ -278,7 +278,7 @@ func (mr *MockControllerMockRecorder) IsDatabaseUpToDate(ctx any) *gomock.Call { } // ListAccounts mocks base method. -func (m *MockController) ListAccounts(ctx context.Context, query ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { +func (m *MockController) ListAccounts(ctx context.Context, query OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListAccounts", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) @@ -293,7 +293,7 @@ func (mr *MockControllerMockRecorder) ListAccounts(ctx, query any) *gomock.Call } // ListLogs mocks base method. -func (m *MockController) ListLogs(ctx context.Context, query GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { +func (m *MockController) ListLogs(ctx context.Context, query ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListLogs", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) @@ -308,7 +308,7 @@ func (mr *MockControllerMockRecorder) ListLogs(ctx, query any) *gomock.Call { } // ListTransactions mocks base method. -func (m *MockController) ListTransactions(ctx context.Context, query ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (m *MockController) ListTransactions(ctx context.Context, query ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ListTransactions", ctx, query) ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) diff --git a/internal/controller/ledger/controller_with_traces.go b/internal/controller/ledger/controller_with_traces.go index b95bb13f0..9bf0aebc8 100644 --- a/internal/controller/ledger/controller_with_traces.go +++ b/internal/controller/ledger/controller_with_traces.go @@ -195,7 +195,7 @@ func (c *ControllerWithTraces) GetMigrationsInfo(ctx context.Context) ([]migrati ) } -func (c *ControllerWithTraces) ListTransactions(ctx context.Context, q ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (c *ControllerWithTraces) ListTransactions(ctx context.Context, q ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Transaction], error) { return tracing.TraceWithMetric( ctx, "ListTransactions", @@ -207,7 +207,7 @@ func (c *ControllerWithTraces) ListTransactions(ctx context.Context, q ListTrans ) } -func (c *ControllerWithTraces) CountTransactions(ctx context.Context, q ListTransactionsQuery) (int, error) { +func (c *ControllerWithTraces) CountTransactions(ctx context.Context, q ResourceQuery[any]) (int, error) { return tracing.TraceWithMetric( ctx, "CountTransactions", @@ -219,7 +219,7 @@ func (c *ControllerWithTraces) CountTransactions(ctx context.Context, q ListTran ) } -func (c *ControllerWithTraces) GetTransaction(ctx context.Context, query GetTransactionQuery) (*ledger.Transaction, error) { +func (c *ControllerWithTraces) GetTransaction(ctx context.Context, query ResourceQuery[any]) (*ledger.Transaction, error) { return tracing.TraceWithMetric( ctx, "GetTransaction", @@ -231,7 +231,7 @@ func (c *ControllerWithTraces) GetTransaction(ctx context.Context, query GetTran ) } -func (c *ControllerWithTraces) CountAccounts(ctx context.Context, a ListAccountsQuery) (int, error) { +func (c *ControllerWithTraces) CountAccounts(ctx context.Context, a ResourceQuery[any]) (int, error) { return tracing.TraceWithMetric( ctx, "CountAccounts", @@ -243,7 +243,7 @@ func (c *ControllerWithTraces) CountAccounts(ctx context.Context, a ListAccounts ) } -func (c *ControllerWithTraces) ListAccounts(ctx context.Context, a ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { +func (c *ControllerWithTraces) ListAccounts(ctx context.Context, a OffsetPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Account], error) { return tracing.TraceWithMetric( ctx, "ListAccounts", @@ -255,7 +255,7 @@ func (c *ControllerWithTraces) ListAccounts(ctx context.Context, a ListAccountsQ ) } -func (c *ControllerWithTraces) GetAccount(ctx context.Context, q GetAccountQuery) (*ledger.Account, error) { +func (c *ControllerWithTraces) GetAccount(ctx context.Context, q ResourceQuery[any]) (*ledger.Account, error) { return tracing.TraceWithMetric( ctx, "GetAccount", @@ -267,7 +267,7 @@ func (c *ControllerWithTraces) GetAccount(ctx context.Context, q GetAccountQuery ) } -func (c *ControllerWithTraces) GetAggregatedBalances(ctx context.Context, q GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { +func (c *ControllerWithTraces) GetAggregatedBalances(ctx context.Context, q ResourceQuery[GetAggregatedVolumesOptions]) (ledger.BalancesByAssets, error) { return tracing.TraceWithMetric( ctx, "GetAggregatedBalances", @@ -279,7 +279,7 @@ func (c *ControllerWithTraces) GetAggregatedBalances(ctx context.Context, q GetA ) } -func (c *ControllerWithTraces) ListLogs(ctx context.Context, q GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { +func (c *ControllerWithTraces) ListLogs(ctx context.Context, q ColumnPaginatedQuery[any]) (*bunpaginate.Cursor[ledger.Log], error) { return tracing.TraceWithMetric( ctx, "ListLogs", @@ -327,7 +327,7 @@ func (c *ControllerWithTraces) IsDatabaseUpToDate(ctx context.Context) (bool, er ) } -func (c *ControllerWithTraces) GetVolumesWithBalances(ctx context.Context, q GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (c *ControllerWithTraces) GetVolumesWithBalances(ctx context.Context, q OffsetPaginatedQuery[GetVolumesOptions]) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { return tracing.TraceWithMetric( ctx, "GetVolumesWithBalances", diff --git a/internal/controller/ledger/stats.go b/internal/controller/ledger/stats.go index 46d9c92c4..24a6a8e84 100644 --- a/internal/controller/ledger/stats.go +++ b/internal/controller/ledger/stats.go @@ -13,12 +13,12 @@ type Stats struct { func (ctrl *DefaultController) GetStats(ctx context.Context) (Stats, error) { var stats Stats - transactions, err := ctrl.store.CountTransactions(ctx, NewListTransactionsQuery(NewPaginatedQueryOptions(PITFilterWithVolumes{}))) + transactions, err := ctrl.store.Transactions().Count(ctx, ResourceQuery[any]{}) if err != nil { return stats, fmt.Errorf("counting transactions: %w", err) } - accounts, err := ctrl.store.CountAccounts(ctx, NewListAccountsQuery(NewPaginatedQueryOptions(PITFilterWithVolumes{}))) + accounts, err := ctrl.store.Accounts().Count(ctx, ResourceQuery[any]{}) if err != nil { return stats, fmt.Errorf("counting accounts: %w", err) } diff --git a/internal/controller/ledger/stats_test.go b/internal/controller/ledger/stats_test.go index e314c6d02..eae3b9801 100644 --- a/internal/controller/ledger/stats_test.go +++ b/internal/controller/ledger/stats_test.go @@ -15,14 +15,13 @@ func TestStats(t *testing.T) { ctrl := gomock.NewController(t) store := NewMockStore(ctrl) parser := NewMockNumscriptParser(ctrl) + transactions := NewMockPaginatedResource[ledger.Transaction, any, ColumnPaginatedQuery[any]](ctrl) + accounts := NewMockPaginatedResource[ledger.Account, any, OffsetPaginatedQuery[any]](ctrl) - store.EXPECT(). - CountTransactions(gomock.Any(), NewListTransactionsQuery(NewPaginatedQueryOptions(PITFilterWithVolumes{}))). - Return(10, nil) - - store.EXPECT(). - CountAccounts(gomock.Any(), NewListAccountsQuery(NewPaginatedQueryOptions(PITFilterWithVolumes{}))). - Return(10, nil) + store.EXPECT().Transactions().Return(transactions) + transactions.EXPECT().Count(ctx, ResourceQuery[any]{}).Return(10, nil) + store.EXPECT().Accounts().Return(accounts) + accounts.EXPECT().Count(ctx, ResourceQuery[any]{}).Return(10, nil) ledgerController := NewDefaultController( ledger.MustNewWithDefault("foo"), diff --git a/internal/controller/ledger/store.go b/internal/controller/ledger/store.go index 9a4579419..3b1375b92 100644 --- a/internal/controller/ledger/store.go +++ b/internal/controller/ledger/store.go @@ -11,7 +11,6 @@ import ( "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/metadata" - "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/go-libs/v2/query" "github.com/formancehq/go-libs/v2/time" ledger "github.com/formancehq/ledger/internal" @@ -53,136 +52,16 @@ type Store interface { LockLedger(ctx context.Context) error GetDB() bun.IDB - ListLogs(ctx context.Context, q GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) ReadLogWithIdempotencyKey(ctx context.Context, ik string) (*ledger.Log, error) - ListTransactions(ctx context.Context, q ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) - CountTransactions(ctx context.Context, q ListTransactionsQuery) (int, error) - GetTransaction(ctx context.Context, query GetTransactionQuery) (*ledger.Transaction, error) - CountAccounts(ctx context.Context, a ListAccountsQuery) (int, error) - ListAccounts(ctx context.Context, a ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) - GetAccount(ctx context.Context, q GetAccountQuery) (*ledger.Account, error) - GetAggregatedBalances(ctx context.Context, q GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) - GetVolumesWithBalances(ctx context.Context, q GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) IsUpToDate(ctx context.Context) (bool, error) GetMigrationsInfo(ctx context.Context) ([]migrations.Info, error) -} - -type ListTransactionsQuery bunpaginate.ColumnPaginatedQuery[PaginatedQueryOptions[PITFilterWithVolumes]] - -func (q ListTransactionsQuery) WithColumn(column string) ListTransactionsQuery { - ret := pointer.For((bunpaginate.ColumnPaginatedQuery[PaginatedQueryOptions[PITFilterWithVolumes]])(q)) - ret = ret.WithColumn(column) - - return ListTransactionsQuery(*ret) -} - -func NewListTransactionsQuery(options PaginatedQueryOptions[PITFilterWithVolumes]) ListTransactionsQuery { - return ListTransactionsQuery{ - PageSize: options.PageSize, - Column: "id", - Order: bunpaginate.OrderDesc, - Options: options, - } -} - -type GetTransactionQuery struct { - PITFilterWithVolumes - ID int -} - -func (q GetTransactionQuery) WithExpandVolumes() GetTransactionQuery { - q.ExpandVolumes = true - - return q -} - -func (q GetTransactionQuery) WithExpandEffectiveVolumes() GetTransactionQuery { - q.ExpandEffectiveVolumes = true - - return q -} - -func NewGetTransactionQuery(id int) GetTransactionQuery { - return GetTransactionQuery{ - PITFilterWithVolumes: PITFilterWithVolumes{}, - ID: id, - } -} - -type ListAccountsQuery bunpaginate.OffsetPaginatedQuery[PaginatedQueryOptions[PITFilterWithVolumes]] - -func (q ListAccountsQuery) WithExpandVolumes() ListAccountsQuery { - q.Options.Options.ExpandVolumes = true - - return q -} - -func (q ListAccountsQuery) WithExpandEffectiveVolumes() ListAccountsQuery { - q.Options.Options.ExpandEffectiveVolumes = true - - return q -} - -func NewListAccountsQuery(opts PaginatedQueryOptions[PITFilterWithVolumes]) ListAccountsQuery { - return ListAccountsQuery{ - PageSize: opts.PageSize, - Order: bunpaginate.OrderAsc, - Options: opts, - } -} - -type GetAccountQuery struct { - PITFilterWithVolumes - Addr string -} - -func (q GetAccountQuery) WithPIT(pit time.Time) GetAccountQuery { - q.PIT = &pit - - return q -} - -func (q GetAccountQuery) WithExpandVolumes() GetAccountQuery { - q.ExpandVolumes = true - - return q -} - -func (q GetAccountQuery) WithExpandEffectiveVolumes() GetAccountQuery { - q.ExpandEffectiveVolumes = true - - return q -} - -func NewGetAccountQuery(addr string) GetAccountQuery { - return GetAccountQuery{ - Addr: addr, - } -} - -type GetAggregatedBalanceQuery struct { - PITFilter - QueryBuilder query.Builder - UseInsertionDate bool -} -func NewGetAggregatedBalancesQuery(filter PITFilter, qb query.Builder, useInsertionDate bool) GetAggregatedBalanceQuery { - return GetAggregatedBalanceQuery{ - PITFilter: filter, - QueryBuilder: qb, - UseInsertionDate: useInsertionDate, - } -} - -type GetVolumesWithBalancesQuery bunpaginate.OffsetPaginatedQuery[PaginatedQueryOptions[FiltersForVolumes]] - -func NewGetVolumesWithBalancesQuery(opts PaginatedQueryOptions[FiltersForVolumes]) GetVolumesWithBalancesQuery { - return GetVolumesWithBalancesQuery{ - PageSize: opts.PageSize, - Order: bunpaginate.OrderAsc, - Options: opts, - } + Accounts() PaginatedResource[ledger.Account, any, OffsetPaginatedQuery[any]] + Logs() PaginatedResource[ledger.Log, any, ColumnPaginatedQuery[any]] + Transactions() PaginatedResource[ledger.Transaction, any, ColumnPaginatedQuery[any]] + AggregatedBalances() Resource[ledger.AggregatedVolumes, GetAggregatedVolumesOptions] + Volumes() PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, OffsetPaginatedQuery[GetVolumesOptions]] } type PaginatedQueryOptions[T any] struct { @@ -237,45 +116,14 @@ func NewPaginatedQueryOptions[T any](options T) PaginatedQueryOptions[T] { } } -type PITFilter struct { - PIT *time.Time `json:"pit"` - OOT *time.Time `json:"oot"` -} - -type PITFilterWithVolumes struct { - PITFilter - ExpandVolumes bool `json:"volumes"` - ExpandEffectiveVolumes bool `json:"effectiveVolumes"` -} - -type FiltersForVolumes struct { - PITFilter - UseInsertionDate bool - GroupLvl int -} - -type GetLogsQuery bunpaginate.ColumnPaginatedQuery[PaginatedQueryOptions[any]] - -func (q GetLogsQuery) WithOrder(order bunpaginate.Order) GetLogsQuery { - q.Order = order - return q -} - -func NewListLogsQuery(options PaginatedQueryOptions[any]) GetLogsQuery { - return GetLogsQuery{ - PageSize: options.PageSize, - Column: "id", - Order: bunpaginate.OrderDesc, - Options: options, - } -} - type vmStoreAdapter struct { Store } func (v *vmStoreAdapter) GetAccount(ctx context.Context, address string) (*ledger.Account, error) { - account, err := v.Store.GetAccount(ctx, NewGetAccountQuery(address)) + account, err := v.Store.Accounts().GetOne(ctx, ResourceQuery[any]{ + Builder: query.Match("address", address), + }) if err != nil { return nil, err } @@ -298,6 +146,75 @@ func NewListLedgersQuery(pageSize uint64) ListLedgersQuery { } } +type ResourceQuery[Opts any] struct { + PIT *time.Time `json:"pit"` + OOT *time.Time `json:"oot"` + Builder query.Builder `json:"qb"` + Expand []string `json:"expand,omitempty"` + Opts Opts `json:"opts"` +} + +func (rq ResourceQuery[Opts]) UsePIT() bool { + return rq.PIT != nil && !rq.PIT.IsZero() +} + +func (rq ResourceQuery[Opts]) UseOOT() bool { + return rq.OOT != nil && !rq.OOT.IsZero() +} + +func (rq *ResourceQuery[Opts]) UnmarshalJSON(data []byte) error { + type rawResourceQuery ResourceQuery[Opts] + type aux struct { + rawResourceQuery + Builder json.RawMessage `json:"qb"` + } + x := aux{} + if err := json.Unmarshal(data, &x); err != nil { + return err + } + + var err error + *rq = ResourceQuery[Opts](x.rawResourceQuery) + rq.Builder, err = query.ParseJSON(string(x.Builder)) + + return err +} + +//go:generate mockgen -write_source_comment=false -write_package_comment=false -source store.go -destination store_generated_test.go -package ledger . Resource +type Resource[ResourceType, OptionsType any] interface { + GetOne(ctx context.Context, query ResourceQuery[OptionsType]) (*ResourceType, error) + Count(ctx context.Context, query ResourceQuery[OptionsType]) (int, error) +} + +type ( + OffsetPaginatedQuery[OptionsType any] struct { + Column string `json:"column"` + Offset uint64 `json:"offset"` + Order *bunpaginate.Order `json:"order"` + PageSize uint64 `json:"pageSize"` + Options ResourceQuery[OptionsType] `json:"filters"` + } + ColumnPaginatedQuery[OptionsType any] struct { + PageSize uint64 `json:"pageSize"` + Bottom *big.Int `json:"bottom"` + Column string `json:"column"` + PaginationID *big.Int `json:"paginationID"` + // todo: backport in go-libs + Order *bunpaginate.Order `json:"order"` + Options ResourceQuery[OptionsType] `json:"filters"` + Reverse bool `json:"reverse"` + } + PaginatedQuery[OptionsType any] interface { + OffsetPaginatedQuery[OptionsType] | ColumnPaginatedQuery[OptionsType] + } +) + +//go:generate mockgen -write_source_comment=false -write_package_comment=false -source store.go -destination store_generated_test.go -package ledger . PaginatedResource +type PaginatedResource[ResourceType, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]] interface { + Resource[ResourceType, OptionsType] + Paginate(ctx context.Context, paginationOptions PaginationQueryType) (*bunpaginate.Cursor[ResourceType], error) +} + // numscript rewrite implementation var _ numscript.Store = (*numscriptRewriteAdapter)(nil) @@ -326,8 +243,8 @@ func (s *numscriptRewriteAdapter) GetAccountsMetadata(ctx context.Context, q num // we ignore the needed metadata values and just return all of them for address := range q { - v, err := s.Store.GetAccount(ctx, GetAccountQuery{ - Addr: address, + v, err := s.Store.Accounts().GetOne(ctx, ResourceQuery[any]{ + Builder: query.Match("address", address), }) if err != nil { return nil, err @@ -337,3 +254,12 @@ func (s *numscriptRewriteAdapter) GetAccountsMetadata(ctx context.Context, q num return m, nil } + +type GetAggregatedVolumesOptions struct { + UseInsertionDate bool `json:"useInsertionDate"` +} + +type GetVolumesOptions struct { + UseInsertionDate bool `json:"useInsertionDate"` + GroupLvl int `json:"groupLvl"` +} diff --git a/internal/controller/ledger/store_generated_test.go b/internal/controller/ledger/store_generated_test.go index 0244559a7..7a677d58e 100644 --- a/internal/controller/ledger/store_generated_test.go +++ b/internal/controller/ledger/store_generated_test.go @@ -2,7 +2,7 @@ // // Generated by this command: // -// mockgen -write_source_comment=false -write_package_comment=false -source store.go -destination store_generated_test.go -package ledger . Store +// mockgen -write_source_comment=false -write_package_comment=false -source store.go -destination store_generated_test.go -package ledger . PaginatedResource package ledger import ( @@ -42,6 +42,34 @@ func (m *MockStore) EXPECT() *MockStoreMockRecorder { return m.recorder } +// Accounts mocks base method. +func (m *MockStore) Accounts() PaginatedResource[ledger.Account, any, OffsetPaginatedQuery[any]] { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Accounts") + ret0, _ := ret[0].(PaginatedResource[ledger.Account, any, OffsetPaginatedQuery[any]]) + return ret0 +} + +// Accounts indicates an expected call of Accounts. +func (mr *MockStoreMockRecorder) Accounts() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Accounts", reflect.TypeOf((*MockStore)(nil).Accounts)) +} + +// AggregatedBalances mocks base method. +func (m *MockStore) AggregatedBalances() Resource[ledger.AggregatedVolumes, GetAggregatedVolumesOptions] { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AggregatedBalances") + ret0, _ := ret[0].(Resource[ledger.AggregatedVolumes, GetAggregatedVolumesOptions]) + return ret0 +} + +// AggregatedBalances indicates an expected call of AggregatedBalances. +func (mr *MockStoreMockRecorder) AggregatedBalances() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AggregatedBalances", reflect.TypeOf((*MockStore)(nil).AggregatedBalances)) +} + // BeginTX mocks base method. func (m *MockStore) BeginTX(ctx context.Context, options *sql.TxOptions) (Store, error) { m.ctrl.T.Helper() @@ -85,36 +113,6 @@ func (mr *MockStoreMockRecorder) CommitTransaction(ctx, transaction any) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CommitTransaction", reflect.TypeOf((*MockStore)(nil).CommitTransaction), ctx, transaction) } -// CountAccounts mocks base method. -func (m *MockStore) CountAccounts(ctx context.Context, a ListAccountsQuery) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CountAccounts", ctx, a) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CountAccounts indicates an expected call of CountAccounts. -func (mr *MockStoreMockRecorder) CountAccounts(ctx, a any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountAccounts", reflect.TypeOf((*MockStore)(nil).CountAccounts), ctx, a) -} - -// CountTransactions mocks base method. -func (m *MockStore) CountTransactions(ctx context.Context, q ListTransactionsQuery) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CountTransactions", ctx, q) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CountTransactions indicates an expected call of CountTransactions. -func (mr *MockStoreMockRecorder) CountTransactions(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CountTransactions", reflect.TypeOf((*MockStore)(nil).CountTransactions), ctx, q) -} - // DeleteAccountMetadata mocks base method. func (m *MockStore) DeleteAccountMetadata(ctx context.Context, address, key string) error { m.ctrl.T.Helper() @@ -145,36 +143,6 @@ func (mr *MockStoreMockRecorder) DeleteTransactionMetadata(ctx, transactionID, k return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteTransactionMetadata", reflect.TypeOf((*MockStore)(nil).DeleteTransactionMetadata), ctx, transactionID, key) } -// GetAccount mocks base method. -func (m *MockStore) GetAccount(ctx context.Context, q GetAccountQuery) (*ledger.Account, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAccount", ctx, q) - ret0, _ := ret[0].(*ledger.Account) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAccount indicates an expected call of GetAccount. -func (mr *MockStoreMockRecorder) GetAccount(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAccount", reflect.TypeOf((*MockStore)(nil).GetAccount), ctx, q) -} - -// GetAggregatedBalances mocks base method. -func (m *MockStore) GetAggregatedBalances(ctx context.Context, q GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetAggregatedBalances", ctx, q) - ret0, _ := ret[0].(ledger.BalancesByAssets) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetAggregatedBalances indicates an expected call of GetAggregatedBalances. -func (mr *MockStoreMockRecorder) GetAggregatedBalances(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAggregatedBalances", reflect.TypeOf((*MockStore)(nil).GetAggregatedBalances), ctx, q) -} - // GetBalances mocks base method. func (m *MockStore) GetBalances(ctx context.Context, query BalanceQuery) (Balances, error) { m.ctrl.T.Helper() @@ -219,36 +187,6 @@ func (mr *MockStoreMockRecorder) GetMigrationsInfo(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetMigrationsInfo", reflect.TypeOf((*MockStore)(nil).GetMigrationsInfo), ctx) } -// GetTransaction mocks base method. -func (m *MockStore) GetTransaction(ctx context.Context, query GetTransactionQuery) (*ledger.Transaction, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetTransaction", ctx, query) - ret0, _ := ret[0].(*ledger.Transaction) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetTransaction indicates an expected call of GetTransaction. -func (mr *MockStoreMockRecorder) GetTransaction(ctx, query any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetTransaction", reflect.TypeOf((*MockStore)(nil).GetTransaction), ctx, query) -} - -// GetVolumesWithBalances mocks base method. -func (m *MockStore) GetVolumesWithBalances(ctx context.Context, q GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVolumesWithBalances", ctx, q) - ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount]) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetVolumesWithBalances indicates an expected call of GetVolumesWithBalances. -func (mr *MockStoreMockRecorder) GetVolumesWithBalances(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVolumesWithBalances", reflect.TypeOf((*MockStore)(nil).GetVolumesWithBalances), ctx, q) -} - // InsertLog mocks base method. func (m *MockStore) InsertLog(ctx context.Context, log *ledger.Log) error { m.ctrl.T.Helper() @@ -278,51 +216,6 @@ func (mr *MockStoreMockRecorder) IsUpToDate(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsUpToDate", reflect.TypeOf((*MockStore)(nil).IsUpToDate), ctx) } -// ListAccounts mocks base method. -func (m *MockStore) ListAccounts(ctx context.Context, a ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListAccounts", ctx, a) - ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Account]) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListAccounts indicates an expected call of ListAccounts. -func (mr *MockStoreMockRecorder) ListAccounts(ctx, a any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListAccounts", reflect.TypeOf((*MockStore)(nil).ListAccounts), ctx, a) -} - -// ListLogs mocks base method. -func (m *MockStore) ListLogs(ctx context.Context, q GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListLogs", ctx, q) - ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Log]) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListLogs indicates an expected call of ListLogs. -func (mr *MockStoreMockRecorder) ListLogs(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLogs", reflect.TypeOf((*MockStore)(nil).ListLogs), ctx, q) -} - -// ListTransactions mocks base method. -func (m *MockStore) ListTransactions(ctx context.Context, q ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListTransactions", ctx, q) - ret0, _ := ret[0].(*bunpaginate.Cursor[ledger.Transaction]) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ListTransactions indicates an expected call of ListTransactions. -func (mr *MockStoreMockRecorder) ListTransactions(ctx, q any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListTransactions", reflect.TypeOf((*MockStore)(nil).ListTransactions), ctx, q) -} - // LockLedger mocks base method. func (m *MockStore) LockLedger(ctx context.Context) error { m.ctrl.T.Helper() @@ -337,6 +230,20 @@ func (mr *MockStoreMockRecorder) LockLedger(ctx any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LockLedger", reflect.TypeOf((*MockStore)(nil).LockLedger), ctx) } +// Logs mocks base method. +func (m *MockStore) Logs() PaginatedResource[ledger.Log, any, ColumnPaginatedQuery[any]] { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Logs") + ret0, _ := ret[0].(PaginatedResource[ledger.Log, any, ColumnPaginatedQuery[any]]) + return ret0 +} + +// Logs indicates an expected call of Logs. +func (mr *MockStoreMockRecorder) Logs() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logs", reflect.TypeOf((*MockStore)(nil).Logs)) +} + // ReadLogWithIdempotencyKey mocks base method. func (m *MockStore) ReadLogWithIdempotencyKey(ctx context.Context, ik string) (*ledger.Log, error) { m.ctrl.T.Helper() @@ -382,6 +289,20 @@ func (mr *MockStoreMockRecorder) Rollback() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rollback", reflect.TypeOf((*MockStore)(nil).Rollback)) } +// Transactions mocks base method. +func (m *MockStore) Transactions() PaginatedResource[ledger.Transaction, any, ColumnPaginatedQuery[any]] { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Transactions") + ret0, _ := ret[0].(PaginatedResource[ledger.Transaction, any, ColumnPaginatedQuery[any]]) + return ret0 +} + +// Transactions indicates an expected call of Transactions. +func (mr *MockStoreMockRecorder) Transactions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Transactions", reflect.TypeOf((*MockStore)(nil).Transactions)) +} + // UpdateAccountsMetadata mocks base method. func (m_2 *MockStore) UpdateAccountsMetadata(ctx context.Context, m map[string]metadata.Metadata) error { m_2.ctrl.T.Helper() @@ -430,3 +351,138 @@ func (mr *MockStoreMockRecorder) UpsertAccounts(ctx any, accounts ...any) *gomoc varargs := append([]any{ctx}, accounts...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpsertAccounts", reflect.TypeOf((*MockStore)(nil).UpsertAccounts), varargs...) } + +// Volumes mocks base method. +func (m *MockStore) Volumes() PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, OffsetPaginatedQuery[GetVolumesOptions]] { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Volumes") + ret0, _ := ret[0].(PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, GetVolumesOptions, OffsetPaginatedQuery[GetVolumesOptions]]) + return ret0 +} + +// Volumes indicates an expected call of Volumes. +func (mr *MockStoreMockRecorder) Volumes() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Volumes", reflect.TypeOf((*MockStore)(nil).Volumes)) +} + +// MockResource is a mock of Resource interface. +type MockResource[ResourceType any, OptionsType any] struct { + ctrl *gomock.Controller + recorder *MockResourceMockRecorder[ResourceType, OptionsType] +} + +// MockResourceMockRecorder is the mock recorder for MockResource. +type MockResourceMockRecorder[ResourceType any, OptionsType any] struct { + mock *MockResource[ResourceType, OptionsType] +} + +// NewMockResource creates a new mock instance. +func NewMockResource[ResourceType any, OptionsType any](ctrl *gomock.Controller) *MockResource[ResourceType, OptionsType] { + mock := &MockResource[ResourceType, OptionsType]{ctrl: ctrl} + mock.recorder = &MockResourceMockRecorder[ResourceType, OptionsType]{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockResource[ResourceType, OptionsType]) EXPECT() *MockResourceMockRecorder[ResourceType, OptionsType] { + return m.recorder +} + +// Count mocks base method. +func (m *MockResource[ResourceType, OptionsType]) Count(ctx context.Context, query ResourceQuery[OptionsType]) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count", ctx, query) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Count indicates an expected call of Count. +func (mr *MockResourceMockRecorder[ResourceType, OptionsType]) Count(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockResource[ResourceType, OptionsType])(nil).Count), ctx, query) +} + +// GetOne mocks base method. +func (m *MockResource[ResourceType, OptionsType]) GetOne(ctx context.Context, query ResourceQuery[OptionsType]) (*ResourceType, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetOne", ctx, query) + ret0, _ := ret[0].(*ResourceType) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetOne indicates an expected call of GetOne. +func (mr *MockResourceMockRecorder[ResourceType, OptionsType]) GetOne(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOne", reflect.TypeOf((*MockResource[ResourceType, OptionsType])(nil).GetOne), ctx, query) +} + +// MockPaginatedResource is a mock of PaginatedResource interface. +type MockPaginatedResource[ResourceType any, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]] struct { + ctrl *gomock.Controller + recorder *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType] +} + +// MockPaginatedResourceMockRecorder is the mock recorder for MockPaginatedResource. +type MockPaginatedResourceMockRecorder[ResourceType any, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]] struct { + mock *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType] +} + +// NewMockPaginatedResource creates a new mock instance. +func NewMockPaginatedResource[ResourceType any, OptionsType any, PaginationQueryType PaginatedQuery[OptionsType]](ctrl *gomock.Controller) *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType] { + mock := &MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]{ctrl: ctrl} + mock.recorder = &MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) EXPECT() *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType] { + return m.recorder +} + +// Count mocks base method. +func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) Count(ctx context.Context, query ResourceQuery[OptionsType]) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Count", ctx, query) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Count indicates an expected call of Count. +func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]) Count(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Count", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType])(nil).Count), ctx, query) +} + +// GetOne mocks base method. +func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) GetOne(ctx context.Context, query ResourceQuery[OptionsType]) (*ResourceType, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetOne", ctx, query) + ret0, _ := ret[0].(*ResourceType) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetOne indicates an expected call of GetOne. +func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]) GetOne(ctx, query any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetOne", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType])(nil).GetOne), ctx, query) +} + +// Paginate mocks base method. +func (m *MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType]) Paginate(ctx context.Context, paginationOptions PaginationQueryType) (*bunpaginate.Cursor[ResourceType], error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Paginate", ctx, paginationOptions) + ret0, _ := ret[0].(*bunpaginate.Cursor[ResourceType]) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Paginate indicates an expected call of Paginate. +func (mr *MockPaginatedResourceMockRecorder[ResourceType, OptionsType, PaginationQueryType]) Paginate(ctx, paginationOptions any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Paginate", reflect.TypeOf((*MockPaginatedResource[ResourceType, OptionsType, PaginationQueryType])(nil).Paginate), ctx, paginationOptions) +} diff --git a/internal/storage/ledger/accounts.go b/internal/storage/ledger/accounts.go index 6fa8545bd..5e28be4ef 100644 --- a/internal/storage/ledger/accounts.go +++ b/internal/storage/ledger/accounts.go @@ -3,320 +3,27 @@ package ledger import ( "context" "fmt" - . "github.com/formancehq/go-libs/v2/bun/bunpaginate" . "github.com/formancehq/go-libs/v2/collectionutils" - "github.com/formancehq/ledger/pkg/features" + "github.com/formancehq/ledger/internal/tracing" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" "regexp" - "github.com/formancehq/ledger/internal/tracing" - "github.com/formancehq/go-libs/v2/metadata" "github.com/formancehq/go-libs/v2/platform/postgres" - "github.com/formancehq/go-libs/v2/time" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - - "github.com/formancehq/go-libs/v2/query" ledger "github.com/formancehq/ledger/internal" - "github.com/uptrace/bun" ) var ( balanceRegex = regexp.MustCompile(`balance\[(.*)]`) ) -func convertOperatorToSQL(operator string) string { - switch operator { - case "$match": - return "=" - case "$lt": - return "<" - case "$gt": - return ">" - case "$lte": - return "<=" - case "$gte": - return ">=" - } - panic("unreachable") -} - -func (s *Store) selectBalance(date *time.Time) (*bun.SelectQuery, error) { - - if date != nil && !date.IsZero() { - selectDistinctMovesBySeq, err := s.SelectDistinctMovesBySeq(date) - if err != nil { - return nil, err - } - sortedMoves := selectDistinctMovesBySeq. - ColumnExpr("(post_commit_volumes).inputs - (post_commit_volumes).outputs as balance") - - return s.db.NewSelect(). - ModelTableExpr("(?) moves", sortedMoves). - Where("ledger = ?", s.ledger.Name). - ColumnExpr("accounts_address, asset, balance"), nil - } - - return s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("accounts_volumes")). - Where("ledger = ?", s.ledger.Name). - ColumnExpr("input - output as balance"), nil -} - -func (s *Store) selectDistinctAccountMetadataHistories(date *time.Time) *bun.SelectQuery { - ret := s.db.NewSelect(). - DistinctOn("accounts_address"). - ModelTableExpr(s.GetPrefixedRelationName("accounts_metadata")). - Where("ledger = ?", s.ledger.Name). - Column("accounts_address", "metadata"). - Order("accounts_address", "revision desc") - - if date != nil && !date.IsZero() { - ret = ret.Where("date <= ?", date) - } - - return ret -} - -func (s *Store) selectAccounts(date *time.Time, expandVolumes, expandEffectiveVolumes bool, qb query.Builder) (*bun.SelectQuery, error) { - - ret := s.db.NewSelect() - - needVolumes := expandVolumes - if qb != nil { - // Analyze filters to check for errors and find potentially additional table to load - if err := qb.Walk(func(operator, key string, value any) error { - switch { - // Balances requires pvc, force load in this case - case balanceRegex.MatchString(key): - needVolumes = true - case key == "address": - return s.validateAddressFilter(operator, value) - case key == "metadata": - if operator != "$exists" { - return ledgercontroller.NewErrInvalidQuery("'metadata' key filter can only be used with $exists") - } - case metadataRegex.MatchString(key): - if operator != "$match" { - return ledgercontroller.NewErrInvalidQuery("'metadata' key filter can only be used with $match") - } - case key == "first_usage" || key == "balance": - default: - return ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) - } - - return nil - }); err != nil { - return nil, fmt.Errorf("failed to check filters: %w", err) - } - } - - // Build the query - ret = ret. - ModelTableExpr(s.GetPrefixedRelationName("accounts")). - Column("accounts.address", "accounts.first_usage"). - Where("ledger = ?", s.ledger.Name). - Order("accounts.address") - - if date != nil && !date.IsZero() { - ret = ret.Where("accounts.first_usage <= ?", date) - } - - if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { - ret = ret. - Join( - `left join (?) accounts_metadata on accounts_metadata.accounts_address = accounts.address`, - s.selectDistinctAccountMetadataHistories(date), - ). - ColumnExpr("coalesce(accounts_metadata.metadata, '{}'::jsonb) as metadata") - } else { - ret = ret.ColumnExpr("accounts.metadata") - } - - if s.ledger.HasFeature(features.FeatureMovesHistory, "ON") && needVolumes { - selectAccountWithAggregatedVolumes, err := s.selectAccountWithAggregatedVolumes(date, true, "volumes") - if err != nil { - return nil, err - } - ret = ret.Join( - `left join (?) volumes on volumes.accounts_address = accounts.address`, - selectAccountWithAggregatedVolumes, - ).Column("volumes.*") - } - - if s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") && expandEffectiveVolumes { - selectAccountWithAggregatedVolumes, err := s.selectAccountWithAggregatedVolumes(date, false, "effective_volumes") - if err != nil { - return nil, err - } - ret = ret.Join( - `left join (?) effective_volumes on effective_volumes.accounts_address = accounts.address`, - selectAccountWithAggregatedVolumes, - ).Column("effective_volumes.*") - } - - if qb != nil { - // Convert filters to where clause - where, args, err := qb.Build(query.ContextFn(func(key, operator string, value any) (string, []any, error) { - switch { - case key == "address": - return filterAccountAddress(value.(string), "accounts.address"), nil, nil - case key == "first_usage": - return fmt.Sprintf("first_usage %s ?", convertOperatorToSQL(operator)), []any{value}, nil - case balanceRegex.Match([]byte(key)): - match := balanceRegex.FindAllStringSubmatch(key, 2) - asset := match[0][1] - - selectBalance, err := s.selectBalance(date) - if err != nil { - return "", nil, err - } - - return s.db.NewSelect(). - TableExpr( - "(?) balance", - selectBalance. - Where("asset = ? and accounts_address = accounts.address", asset), - ). - ColumnExpr(fmt.Sprintf("balance %s ?", convertOperatorToSQL(operator)), value). - String(), nil, nil - - case key == "balance": - selectBalance, err := s.selectBalance(date) - if err != nil { - return "", nil, err - } - - return s.db.NewSelect(). - TableExpr( - "(?) balance", - selectBalance. - Where("accounts_address = accounts.address"), - ). - ColumnExpr(fmt.Sprintf("balance %s ?", convertOperatorToSQL(operator)), value). - String(), nil, nil - - case key == "metadata": - if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { - key = "accounts_metadata.metadata" - } - - return key + " -> ? is not null", []any{value}, nil - - case metadataRegex.Match([]byte(key)): - match := metadataRegex.FindAllStringSubmatch(key, 3) - if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { - key = "accounts_metadata.metadata" - } else { - key = "metadata" - } - - return key + " @> ?", []any{map[string]any{ - match[0][1]: value, - }}, nil - } - - panic("unreachable") - })) - if err != nil { - return nil, fmt.Errorf("evaluating filters: %w", err) - } - if len(args) > 0 { - ret = ret.Where(where, args...) - } else { - ret = ret.Where(where) - } - } - - return ret, nil -} - -func (s *Store) ListAccounts(ctx context.Context, q ledgercontroller.ListAccountsQuery) (*Cursor[ledger.Account], error) { - selectAccounts, err := s.selectAccounts( - q.Options.Options.PIT, - q.Options.Options.ExpandVolumes, - q.Options.Options.ExpandEffectiveVolumes, - q.Options.QueryBuilder, - ) - if err != nil { - return nil, err - } - return tracing.TraceWithMetric( - ctx, - "ListAccounts", - s.tracer, - s.listAccountsHistogram, - func(ctx context.Context) (*Cursor[ledger.Account], error) { - ret, err := UsingOffset[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes], ledger.Account]( - ctx, - selectAccounts, - OffsetPaginatedQuery[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes]](q), - ) - - if err != nil { - return nil, err - } - - return ret, nil - }, - ) -} - -func (s *Store) GetAccount(ctx context.Context, q ledgercontroller.GetAccountQuery) (*ledger.Account, error) { - return tracing.TraceWithMetric( - ctx, - "GetAccount", - s.tracer, - s.getAccountHistogram, - func(ctx context.Context) (*ledger.Account, error) { - ret := &ledger.Account{} - selectAccounts, err := s.selectAccounts(q.PIT, q.ExpandVolumes, q.ExpandEffectiveVolumes, nil) - if err != nil { - return nil, err - } - if err := selectAccounts. - Model(ret). - Where("accounts.address = ?", q.Addr). - Limit(1). - Scan(ctx); err != nil { - return nil, postgres.ResolveError(err) - } - - return ret, nil - }, - ) -} - -func (s *Store) CountAccounts(ctx context.Context, q ledgercontroller.ListAccountsQuery) (int, error) { - return tracing.TraceWithMetric( - ctx, - "CountAccounts", - s.tracer, - s.countAccountsHistogram, - func(ctx context.Context) (int, error) { - selectAccounts, err := s.selectAccounts( - q.Options.Options.PIT, - q.Options.Options.ExpandVolumes, - q.Options.Options.ExpandEffectiveVolumes, - q.Options.QueryBuilder, - ) - if err != nil { - return 0, err - } - return s.db.NewSelect(). - TableExpr("(?) data", selectAccounts). - Count(ctx) - }, - ) -} - -func (s *Store) UpdateAccountsMetadata(ctx context.Context, m map[string]metadata.Metadata) error { +func (store *Store) UpdateAccountsMetadata(ctx context.Context, m map[string]metadata.Metadata) error { _, err := tracing.TraceWithMetric( ctx, "UpdateAccountsMetadata", - s.tracer, - s.updateAccountsMetadataHistogram, + store.tracer, + store.updateAccountsMetadataHistogram, tracing.NoResult(func(ctx context.Context) error { span := trace.SpanFromContext(ctx) @@ -330,7 +37,7 @@ func (s *Store) UpdateAccountsMetadata(ctx context.Context, m map[string]metadat accounts := make([]AccountWithLedger, 0) for account, accountMetadata := range m { accounts = append(accounts, AccountWithLedger{ - Ledger: s.ledger.Name, + Ledger: store.ledger.Name, Account: ledger.Account{ Address: account, Metadata: accountMetadata, @@ -338,11 +45,12 @@ func (s *Store) UpdateAccountsMetadata(ctx context.Context, m map[string]metadat }) } - ret, err := s.db.NewInsert(). + ret, err := store.db.NewInsert(). Model(&accounts). - ModelTableExpr(s.GetPrefixedRelationName("accounts")). + ModelTableExpr(store.GetPrefixedRelationName("accounts")). On("CONFLICT (ledger, address) DO UPDATE"). Set("metadata = excluded.metadata || accounts.metadata"). + Set("updated_at = excluded.updated_at"). Where("not accounts.metadata @> excluded.metadata"). Exec(ctx) @@ -363,18 +71,18 @@ func (s *Store) UpdateAccountsMetadata(ctx context.Context, m map[string]metadat return err } -func (s *Store) DeleteAccountMetadata(ctx context.Context, account, key string) error { +func (store *Store) DeleteAccountMetadata(ctx context.Context, account, key string) error { _, err := tracing.TraceWithMetric( ctx, "DeleteAccountMetadata", - s.tracer, - s.deleteAccountMetadataHistogram, + store.tracer, + store.deleteAccountMetadataHistogram, tracing.NoResult(func(ctx context.Context) error { - _, err := s.db.NewUpdate(). - ModelTableExpr(s.GetPrefixedRelationName("accounts")). + _, err := store.db.NewUpdate(). + ModelTableExpr(store.GetPrefixedRelationName("accounts")). Set("metadata = metadata - ?", key). Where("address = ?", account). - Where("ledger = ?", s.ledger.Name). + Where("ledger = ?", store.ledger.Name). Exec(ctx) return postgres.ResolveError(err) }), @@ -382,24 +90,24 @@ func (s *Store) DeleteAccountMetadata(ctx context.Context, account, key string) return err } -func (s *Store) UpsertAccounts(ctx context.Context, accounts ...*ledger.Account) error { +func (store *Store) UpsertAccounts(ctx context.Context, accounts ...*ledger.Account) error { return tracing.SkipResult(tracing.TraceWithMetric( ctx, "UpsertAccounts", - s.tracer, - s.upsertAccountsHistogram, + store.tracer, + store.upsertAccountsHistogram, tracing.NoResult(func(ctx context.Context) error { span := trace.SpanFromContext(ctx) span.SetAttributes(attribute.StringSlice("accounts", Map(accounts, (*ledger.Account).GetAddress))) - ret, err := s.db.NewInsert(). + ret, err := store.db.NewInsert(). Model(&accounts). - ModelTableExpr(s.GetPrefixedRelationName("accounts")). + ModelTableExpr(store.GetPrefixedRelationName("accounts")). On("conflict (ledger, address) do update"). Set("first_usage = case when excluded.first_usage < accounts.first_usage then excluded.first_usage else accounts.first_usage end"). Set("metadata = accounts.metadata || excluded.metadata"). Set("updated_at = excluded.updated_at"). - Value("ledger", "?", s.ledger.Name). + Value("ledger", "?", store.ledger.Name). Returning("*"). Where("(excluded.first_usage < accounts.first_usage) or not accounts.metadata @> excluded.metadata"). Exec(ctx) diff --git a/internal/storage/ledger/accounts_test.go b/internal/storage/ledger/accounts_test.go index ef0c2f6d1..4d20e16d6 100644 --- a/internal/storage/ledger/accounts_test.go +++ b/internal/storage/ledger/accounts_test.go @@ -6,6 +6,7 @@ import ( "context" "math/big" "testing" + libtime "time" "errors" "github.com/formancehq/go-libs/v2/pointer" @@ -71,27 +72,29 @@ func TestAccountsList(t *testing.T) { t.Run("list all", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}))) + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{}) require.NoError(t, err) require.Len(t, accounts.Data, 7) }) t.Run("list using metadata", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("metadata[category]", "1")), - )) + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("metadata[category]", "1"), + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) }) t.Run("list before date", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ PIT: &now, }, - }))) + }) require.NoError(t, err) require.Len(t, accounts.Data, 2) }) @@ -99,9 +102,12 @@ func TestAccountsList(t *testing.T) { t.Run("list with volumes", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - ExpandVolumes: true, - }).WithQueryBuilder(query.Match("address", "account:1")))) + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "account:1"), + Expand: []string{"volumes"}, + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) require.Equal(t, ledger.VolumesByAssets{ @@ -112,12 +118,13 @@ func TestAccountsList(t *testing.T) { t.Run("list with volumes using PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "account:1"), + PIT: &now, + Expand: []string{"volumes"}, }, - ExpandVolumes: true, - }).WithQueryBuilder(query.Match("address", "account:1")))) + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) require.Equal(t, ledger.VolumesByAssets{ @@ -128,9 +135,12 @@ func TestAccountsList(t *testing.T) { t.Run("list with effective volumes", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - ExpandEffectiveVolumes: true, - }).WithQueryBuilder(query.Match("address", "account:1")))) + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "account:1"), + Expand: []string{"effectiveVolumes"}, + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) require.Equal(t, ledger.VolumesByAssets{ @@ -140,12 +150,13 @@ func TestAccountsList(t *testing.T) { t.Run("list with effective volumes using PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "account:1"), + PIT: &now, + Expand: []string{"effectiveVolumes"}, }, - ExpandEffectiveVolumes: true, - }).WithQueryBuilder(query.Match("address", "account:1")))) + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) require.Equal(t, ledger.VolumesByAssets{ @@ -155,36 +166,42 @@ func TestAccountsList(t *testing.T) { t.Run("list using filter on address", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("address", "account:")), - )) + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "account:"), + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 3) }) t.Run("list using filter on multiple address", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder( - query.Or( + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Or( query.Match("address", "account:1"), query.Match("address", "orders:"), ), - ), - )) + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 3) }) t.Run("list using filter on balances", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Lt("balance[USD]", 0)), - )) + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("balance[USD]", 0), + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world - accounts, err = store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Gt("balance[USD]", 0)), - )) + accounts, err = store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Gt("balance[USD]", 0), + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 2) require.Equal(t, "account:1", accounts.Data[0].Address) @@ -192,49 +209,53 @@ func TestAccountsList(t *testing.T) { }) t.Run("list using filter on balances[USD] and PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("balance[USD]", 0), + PIT: &now, }, - }). - WithQueryBuilder(query.Lt("balance[USD]", 0)), - )) + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world }) t.Run("list using filter on balances and PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &now, + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("balance", 0), + PIT: &now, }, - }). - WithQueryBuilder(query.Lt("balance", 0)), - )) + }) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world }) t.Run("list using filter on exists metadata", func(t *testing.T) { t.Parallel() - accounts, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Exists("metadata", "foo")), - )) + accounts, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Exists("metadata", "foo"), + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 2) - accounts, err = store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Exists("metadata", "category")), - )) + accounts, err = store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Exists("metadata", "category"), + }, + }) require.NoError(t, err) require.Len(t, accounts.Data, 3) }) t.Run("list using filter invalid field", func(t *testing.T) { t.Parallel() - _, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Lt("invalid", 0)), - )) + _, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lt("invalid", 0), + }, + }) require.Error(t, err) require.True(t, errors.Is(err, ledgercontroller.ErrInvalidQuery{})) }) @@ -242,9 +263,11 @@ func TestAccountsList(t *testing.T) { t.Run("filter on first_usage", func(t *testing.T) { t.Parallel() - ret, err := store.ListAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Lt("first_usage", now)), - )) + ret, err := store.Accounts().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Lte("first_usage", now), + }, + }) require.NoError(t, err) require.Len(t, ret.Data, 2) }) @@ -263,7 +286,9 @@ func TestAccountsUpdateMetadata(t *testing.T) { "bank": m, })) - account, err := store.GetAccount(context.Background(), ledgercontroller.NewGetAccountQuery("bank")) + account, err := store.Accounts().GetOne(context.Background(), ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "bank"), + }) require.NoError(t, err, "account retrieval should not fail") require.Equal(t, "bank", account.Address, "account address should match") @@ -277,59 +302,79 @@ func TestAccountsGet(t *testing.T) { now := time.Now() ctx := logging.TestingContext() - err := store.CommitTransaction(ctx, pointer.For(ledger.NewTransaction().WithPostings( + tx1 := pointer.For(ledger.NewTransaction().WithPostings( ledger.NewPosting("world", "multi", "USD/2", big.NewInt(100)), - ).WithTimestamp(now))) + ).WithTimestamp(now)) + err := store.CommitTransaction(ctx, tx1) require.NoError(t, err) + // sleep for at least the time precision to ensure the next transaction is inserted with a different timestamp + libtime.Sleep(time.DatePrecision) + require.NoError(t, store.UpdateAccountsMetadata(ctx, map[string]metadata.Metadata{ "multi": { "category": "gold", }, })) - err = store.CommitTransaction(ctx, pointer.For(ledger.NewTransaction().WithPostings( + tx2 := pointer.For(ledger.NewTransaction().WithPostings( ledger.NewPosting("world", "multi", "USD/2", big.NewInt(0)), - ).WithTimestamp(now.Add(-time.Minute)))) + ).WithTimestamp(now.Add(-time.Minute))) + err = store.CommitTransaction(ctx, tx2) require.NoError(t, err) t.Run("find account", func(t *testing.T) { t.Parallel() - account, err := store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("multi")) + account, err := store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "multi"), + }) require.NoError(t, err) require.Equal(t, ledger.Account{ Address: "multi", Metadata: metadata.Metadata{ "category": "gold", }, - FirstUsage: now.Add(-time.Minute), + FirstUsage: now.Add(-time.Minute), + InsertionDate: tx1.InsertedAt, + UpdatedAt: tx2.InsertedAt, }, *account) - account, err = store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("world")) + account, err = store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "world"), + }) require.NoError(t, err) require.Equal(t, ledger.Account{ - Address: "world", - Metadata: metadata.Metadata{}, - FirstUsage: now.Add(-time.Minute), + Address: "world", + Metadata: metadata.Metadata{}, + FirstUsage: now.Add(-time.Minute), + InsertionDate: tx1.InsertedAt, + UpdatedAt: tx2.InsertedAt, }, *account) }) t.Run("find account in past", func(t *testing.T) { t.Parallel() - account, err := store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("multi").WithPIT(now.Add(-30*time.Second))) + account, err := store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "multi"), + PIT: pointer.For(now.Add(-30 * time.Second)), + }) require.NoError(t, err) require.Equal(t, ledger.Account{ - Address: "multi", - Metadata: metadata.Metadata{}, - FirstUsage: now.Add(-time.Minute), + Address: "multi", + Metadata: metadata.Metadata{}, + FirstUsage: now.Add(-time.Minute), + InsertionDate: tx1.InsertedAt, + UpdatedAt: tx2.InsertedAt, }, *account) }) t.Run("find account with volumes", func(t *testing.T) { t.Parallel() - account, err := store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("multi"). - WithExpandVolumes()) + account, err := store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "multi"), + Expand: []string{"volumes"}, + }) require.NoError(t, err) require.Equal(t, ledger.Account{ Address: "multi", @@ -340,13 +385,17 @@ func TestAccountsGet(t *testing.T) { Volumes: ledger.VolumesByAssets{ "USD/2": ledger.NewVolumesInt64(100, 0), }, + InsertionDate: tx1.InsertedAt, + UpdatedAt: tx2.InsertedAt, }, *account) }) t.Run("find account with effective volumes", func(t *testing.T) { t.Parallel() - account, err := store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("multi"). - WithExpandEffectiveVolumes()) + account, err := store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "multi"), + Expand: []string{"effectiveVolumes"}, + }) require.NoError(t, err) require.Equal(t, ledger.Account{ Address: "multi", @@ -357,25 +406,34 @@ func TestAccountsGet(t *testing.T) { EffectiveVolumes: ledger.VolumesByAssets{ "USD/2": ledger.NewVolumesInt64(100, 0), }, + InsertionDate: tx1.InsertedAt, + UpdatedAt: tx2.InsertedAt, }, *account) }) t.Run("find account using pit", func(t *testing.T) { t.Parallel() - account, err := store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("multi").WithPIT(now)) + account, err := store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "multi"), + PIT: pointer.For(now), + }) require.NoError(t, err) require.Equal(t, ledger.Account{ - Address: "multi", - Metadata: metadata.Metadata{}, - FirstUsage: now.Add(-time.Minute), + Address: "multi", + Metadata: metadata.Metadata{}, + FirstUsage: now.Add(-time.Minute), + InsertionDate: tx1.InsertedAt, + UpdatedAt: tx2.InsertedAt, }, *account) }) t.Run("not existent account", func(t *testing.T) { t.Parallel() - _, err := store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("account_not_existing")) + _, err := store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "account_not_existing"), + }) require.Error(t, err) }) } @@ -391,7 +449,7 @@ func TestAccountsCount(t *testing.T) { ))) require.NoError(t, err) - countAccounts, err := store.CountAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}))) + countAccounts, err := store.Accounts().Count(ctx, ledgercontroller.ResourceQuery[any]{}) require.NoError(t, err) require.EqualValues(t, 2, countAccounts) // world + central_bank } diff --git a/internal/storage/ledger/balances.go b/internal/storage/ledger/balances.go index adbb07950..d702f7135 100644 --- a/internal/storage/ledger/balances.go +++ b/internal/storage/ledger/balances.go @@ -2,8 +2,6 @@ package ledger import ( "context" - "fmt" - "github.com/formancehq/ledger/pkg/features" "math/big" "strings" @@ -11,207 +9,23 @@ import ( "github.com/formancehq/ledger/internal/tracing" - "github.com/formancehq/go-libs/v2/query" - "github.com/formancehq/go-libs/v2/time" ledger "github.com/formancehq/ledger/internal" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/uptrace/bun" ) -func (s *Store) selectAccountWithAssetAndVolumes(date *time.Time, useInsertionDate bool, builder query.Builder) (*bun.SelectQuery, error) { - - var ( - needMetadata bool - needAddressSegment bool - ) - - if builder != nil { - if err := builder.Walk(func(operator string, key string, value any) error { - switch { - case key == "address": - if err := s.validateAddressFilter(operator, value); err != nil { - return err - } - if !needAddressSegment { - // Cast is safe, the type has been validated by validatedAddressFilter - needAddressSegment = isSegmentedAddress(value.(string)) - } - - case key == "metadata": - needMetadata = true - if operator != "$exists" { - return ledgercontroller.NewErrInvalidQuery("'metadata' key filter can only be used with $exists") - } - case metadataRegex.Match([]byte(key)): - needMetadata = true - if operator != "$match" { - return ledgercontroller.NewErrInvalidQuery("'account' column can only be used with $match") - } - default: - return ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) - } - return nil - }); err != nil { - return nil, err - } - } - - if needAddressSegment && !s.ledger.HasFeature(features.FeatureIndexAddressSegments, "ON") { - return nil, ledgercontroller.NewErrMissingFeature(features.FeatureIndexAddressSegments) - } - - var selectAccountsWithVolumes *bun.SelectQuery - if date != nil && !date.IsZero() { - if useInsertionDate { - if !s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { - return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory) - } - selectDistinctMovesBySeq, err := s.SelectDistinctMovesBySeq(date) - if err != nil { - return nil, err - } - selectAccountsWithVolumes = s.db.NewSelect(). - TableExpr("(?) moves", selectDistinctMovesBySeq). - Column("asset", "accounts_address"). - ColumnExpr("post_commit_volumes as volumes") - } else { - if !s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { - return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes) - } - selectAccountsWithVolumes = s.db.NewSelect(). - TableExpr("(?) moves", s.SelectDistinctMovesByEffectiveDate(date)). - Column("asset", "accounts_address"). - ColumnExpr("moves.post_commit_effective_volumes as volumes") - } - } else { - selectAccountsWithVolumes = s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("accounts_volumes")). - Column("asset", "accounts_address"). - ColumnExpr("(input, output)::"+s.GetPrefixedRelationName("volumes")+" as volumes"). - Where("ledger = ?", s.ledger.Name) - } - - selectAccountsWithVolumes = s.db.NewSelect(). - ColumnExpr("*"). - TableExpr("(?) accounts_volumes", selectAccountsWithVolumes) - - needAccount := needAddressSegment - if needMetadata { - if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { - selectAccountsWithVolumes = selectAccountsWithVolumes. - Join( - `left join (?) accounts_metadata on accounts_metadata.accounts_address = accounts_volumes.accounts_address`, - s.selectDistinctAccountMetadataHistories(date), - ) - } else { - needAccount = true - } - } - - if needAccount { - selectAccountsWithVolumes = s.db.NewSelect(). - TableExpr( - "(?) accounts", - selectAccountsWithVolumes. - Join("join "+s.GetPrefixedRelationName("accounts")+" accounts on accounts.address = accounts_volumes.accounts_address and ledger = ?", s.ledger.Name), - ). - ColumnExpr("address, asset, volumes, metadata"). - ColumnExpr("accounts.address_array as accounts_address_array") - } - - finalQuery := s.db.NewSelect(). - TableExpr("(?) accounts", selectAccountsWithVolumes) - - if builder != nil { - where, args, err := builder.Build(query.ContextFn(func(key, _ string, value any) (string, []any, error) { - switch { - case key == "address": - return filterAccountAddress(value.(string), "accounts_address"), nil, nil - case metadataRegex.Match([]byte(key)): - match := metadataRegex.FindAllStringSubmatch(key, 3) - - return "metadata @> ?", []any{map[string]any{ - match[0][1]: value, - }}, nil - - case key == "metadata": - return "metadata -> ? is not null", []any{value}, nil - default: - return "", nil, ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) - } - })) - if err != nil { - return nil, fmt.Errorf("building where clause: %w", err) - } - finalQuery = finalQuery.Where(where, args...) - } - - return finalQuery, nil -} - -func (s *Store) selectAccountWithAggregatedVolumes(date *time.Time, useInsertionDate bool, alias string) (*bun.SelectQuery, error) { - selectAccountWithAssetAndVolumes, err := s.selectAccountWithAssetAndVolumes(date, useInsertionDate, nil) - if err != nil { - return nil, err - } - return s.db.NewSelect(). - TableExpr("(?) values", selectAccountWithAssetAndVolumes). - Group("accounts_address"). - Column("accounts_address"). - ColumnExpr("public.aggregate_objects(json_build_object(asset, json_build_object('input', (volumes).inputs, 'output', (volumes).outputs))::jsonb) as " + alias), nil -} - -func (s *Store) SelectAggregatedBalances(date *time.Time, useInsertionDate bool, builder query.Builder) (*bun.SelectQuery, error) { - - selectAccountsWithVolumes, err := s.selectAccountWithAssetAndVolumes(date, useInsertionDate, builder) - if err != nil { - return nil, err - } - sumVolumesForAsset := s.db.NewSelect(). - TableExpr("(?) values", selectAccountsWithVolumes). - Group("asset"). - Column("asset"). - ColumnExpr("json_build_object('input', sum(((volumes).inputs)::numeric), 'output', sum(((volumes).outputs)::numeric)) as volumes") - - return s.db.NewSelect(). - TableExpr("(?) values", sumVolumesForAsset). - ColumnExpr("aggregate_objects(json_build_object(asset, volumes)::jsonb) as aggregated"), nil -} - -func (s *Store) GetAggregatedBalances(ctx context.Context, q ledgercontroller.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { - type AggregatedVolumes struct { - Aggregated ledger.VolumesByAssets `bun:"aggregated,type:jsonb"` - } - - selectAggregatedBalances, err := s.SelectAggregatedBalances(q.PIT, q.UseInsertionDate, q.QueryBuilder) - if err != nil { - return nil, err - } - - aggregatedVolumes := AggregatedVolumes{} - if err := s.db.NewSelect(). - ModelTableExpr("(?) aggregated_volumes", selectAggregatedBalances). - Model(&aggregatedVolumes). - Scan(ctx); err != nil { - return nil, err - } - - return aggregatedVolumes.Aggregated.Balances(), nil -} - -func (s *Store) GetBalances(ctx context.Context, query ledgercontroller.BalanceQuery) (ledgercontroller.Balances, error) { +func (store *Store) GetBalances(ctx context.Context, query ledgercontroller.BalanceQuery) (ledgercontroller.Balances, error) { return tracing.TraceWithMetric( ctx, "GetBalances", - s.tracer, - s.getBalancesHistogram, + store.tracer, + store.getBalancesHistogram, func(ctx context.Context) (ledgercontroller.Balances, error) { conditions := make([]string, 0) args := make([]any, 0) for account, assets := range query { for _, asset := range assets { conditions = append(conditions, "ledger = ? and accounts_address = ? and asset = ?") - args = append(args, s.ledger.Name, account, asset) + args = append(args, store.ledger.Name, account, asset) } } @@ -224,7 +38,7 @@ func (s *Store) GetBalances(ctx context.Context, query ledgercontroller.BalanceQ for account, assets := range query { for _, asset := range assets { accountsVolumes = append(accountsVolumes, AccountsVolumesWithLedger{ - Ledger: s.ledger.Name, + Ledger: store.ledger.Name, AccountsVolumes: ledger.AccountsVolumes{ Account: account, Asset: asset, @@ -238,55 +52,55 @@ func (s *Store) GetBalances(ctx context.Context, query ledgercontroller.BalanceQ // Try to insert volumes using last move (to keep compat with previous version) or 0 values. // This way, if the account has a 0 balance at this point, it will be locked as any other accounts. // If the complete sql transaction fails, the account volumes will not be inserted. - selectMoves := s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("moves")). + selectMoves := store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName("moves")). DistinctOn("accounts_address, asset"). Column("accounts_address", "asset"). ColumnExpr("first_value(post_commit_volumes) over (partition by accounts_address, asset order by seq desc) as post_commit_volumes"). ColumnExpr("first_value(ledger) over (partition by accounts_address, asset order by seq desc) as ledger"). Where("("+strings.Join(conditions, ") OR (")+")", args...) - zeroValuesAndMoves := s.db.NewSelect(). + zeroValuesAndMoves := store.db.NewSelect(). TableExpr("(?) data", selectMoves). Column("ledger", "accounts_address", "asset"). ColumnExpr("(post_commit_volumes).inputs as input"). ColumnExpr("(post_commit_volumes).outputs as output"). UnionAll( - s.db.NewSelect(). + store.db.NewSelect(). TableExpr( "(?) data", - s.db.NewSelect().NewValues(&accountsVolumes), + store.db.NewSelect().NewValues(&accountsVolumes), ). Column("*"), ) - zeroValueOrMoves := s.db.NewSelect(). + zeroValueOrMoves := store.db.NewSelect(). TableExpr("(?) data", zeroValuesAndMoves). Column("ledger", "accounts_address", "asset", "input", "output"). DistinctOn("ledger, accounts_address, asset") - insertDefaultValue := s.db.NewInsert(). - TableExpr(s.GetPrefixedRelationName("accounts_volumes")). + insertDefaultValue := store.db.NewInsert(). + TableExpr(store.GetPrefixedRelationName("accounts_volumes")). TableExpr("(" + zeroValueOrMoves.String() + ") data"). On("conflict (ledger, accounts_address, asset) do nothing"). Returning("ledger, accounts_address, asset, input, output") - selectExistingValues := s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("accounts_volumes")). + selectExistingValues := store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName("accounts_volumes")). Column("ledger", "accounts_address", "asset", "input", "output"). Where("("+strings.Join(conditions, ") OR (")+")", args...). For("update"). // notes(gfyrag): Keep order, it ensures consistent locking order and limit deadlocks Order("accounts_address", "asset") - finalQuery := s.db.NewSelect(). + finalQuery := store.db.NewSelect(). With("inserted", insertDefaultValue). With("existing", selectExistingValues). ModelTableExpr( "(?) accounts_volumes", - s.db.NewSelect(). + store.db.NewSelect(). ModelTableExpr("inserted"). - UnionAll(s.db.NewSelect().ModelTableExpr("existing")), + UnionAll(store.db.NewSelect().ModelTableExpr("existing")), ). Model(&accountsVolumes) diff --git a/internal/storage/ledger/balances_test.go b/internal/storage/ledger/balances_test.go index 3507962df..3713294e9 100644 --- a/internal/storage/ledger/balances_test.go +++ b/internal/storage/ledger/balances_test.go @@ -239,106 +239,168 @@ func TestBalancesAggregates(t *testing.T) { t.Run("aggregate on all", func(t *testing.T) { t.Parallel() - cursor, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, nil, false)) + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{}) require.NoError(t, err) - RequireEqual(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0), - "EUR": big.NewInt(0), - }, cursor) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Add( + big.NewInt(0).Mul(bigInt, big.NewInt(2)), + big.NewInt(0).Mul(smallInt, big.NewInt(2)), + ), + Output: big.NewInt(0).Add( + big.NewInt(0).Mul(bigInt, big.NewInt(2)), + big.NewInt(0).Mul(smallInt, big.NewInt(2)), + ), + }, + "EUR": ledger.Volumes{ + Input: smallInt, + Output: smallInt, + }, + }, + }, *ret) }) t.Run("filter on address", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, - query.Match("address", "users:"), false)) + + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Builder: query.Match("address", "users:"), + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0).Add( - big.NewInt(0).Mul(bigInt, big.NewInt(2)), - big.NewInt(0).Mul(smallInt, big.NewInt(2)), - ), - }, ret) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Add( + big.NewInt(0).Mul(bigInt, big.NewInt(2)), + big.NewInt(0).Mul(smallInt, big.NewInt(2)), + ), + Output: new(big.Int), + }, + }, + }, *ret) }) t.Run("using pit on effective date", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{ - PIT: pointer.For(now.Add(-time.Second)), - }, query.Match("address", "users:"), false)) + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Builder: query.Match("address", "users:"), + PIT: pointer.For(now.Add(-time.Second)), + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0).Add( - bigInt, - smallInt, - ), - }, ret) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Add( + bigInt, + smallInt, + ), + Output: new(big.Int), + }, + }, + }, *ret) }) t.Run("using pit on insertion date", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{ - PIT: pointer.For(now), - }, query.Match("address", "users:"), true)) + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Builder: query.Match("address", "users:"), + PIT: pointer.For(now), + Opts: ledgercontroller.GetAggregatedVolumesOptions{ + UseInsertionDate: true, + }, + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0).Add( - bigInt, - smallInt, - ), - }, ret) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Add( + bigInt, + smallInt, + ), + Output: new(big.Int), + }, + }, + }, *ret) }) t.Run("using a metadata and pit", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{ - PIT: pointer.For(now.Add(time.Minute)), - }, query.Match("metadata[category]", "premium"), false)) + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + PIT: pointer.For(now.Add(time.Minute)), + Builder: query.Match("metadata[category]", "premium"), + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0).Add( - big.NewInt(0).Mul(bigInt, big.NewInt(2)), - big.NewInt(0), - ), - }, ret) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Add( + big.NewInt(0).Mul(bigInt, big.NewInt(2)), + big.NewInt(0), + ), + Output: new(big.Int), + }, + }, + }, *ret) }) t.Run("using a metadata without pit", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, - query.Match("metadata[category]", "premium"), false)) + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Builder: query.Match("metadata[category]", "premium"), + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0).Mul(bigInt, big.NewInt(2)), - }, ret) + + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Mul(bigInt, big.NewInt(2)), + Output: new(big.Int), + }, + }, + }, *ret) }) t.Run("when no matching", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, - query.Match("metadata[category]", "guest"), false)) + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Builder: query.Match("metadata[category]", "guest"), + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{}, ret) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{}, + }, *ret) }) t.Run("using a filter exist on metadata", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, query.Exists("metadata", "category"), false)) + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Builder: query.Exists("metadata", "category"), + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0).Add( - big.NewInt(0).Mul(bigInt, big.NewInt(2)), - big.NewInt(0).Mul(smallInt, big.NewInt(2)), - ), - }, ret) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Add( + big.NewInt(0).Mul(bigInt, big.NewInt(2)), + big.NewInt(0).Mul(smallInt, big.NewInt(2)), + ), + Output: new(big.Int), + }, + }, + }, *ret) }) t.Run("using a filter on metadata and on address", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery( - ledgercontroller.PITFilter{}, - query.And( + ret, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Builder: query.And( query.Match("address", "users:"), query.Match("metadata[category]", "premium"), ), - false, - )) + }) require.NoError(t, err) - require.Equal(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0).Mul(bigInt, big.NewInt(2)), - }, ret) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": ledger.Volumes{ + Input: big.NewInt(0).Mul(bigInt, big.NewInt(2)), + Output: new(big.Int), + }, + }, + }, *ret) }) } diff --git a/internal/storage/ledger/debug.go b/internal/storage/ledger/debug.go index 4cb9c4689..79e79fa64 100644 --- a/internal/storage/ledger/debug.go +++ b/internal/storage/ledger/debug.go @@ -9,28 +9,28 @@ import ( ) //nolint:unused -func (s *Store) DumpTables(ctx context.Context, tables ...string) { +func (store *Store) DumpTables(ctx context.Context, tables ...string) { for _, table := range tables { - s.DumpQuery( + store.DumpQuery( ctx, - s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName(table)), + store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName(table)), ) } } //nolint:unused -func (s *Store) DumpQuery(ctx context.Context, query *bun.SelectQuery) { +func (store *Store) DumpQuery(ctx context.Context, query *bun.SelectQuery) { fmt.Println(query) rows, err := query.Rows(ctx) if err != nil { panic(err) } - s.DumpRows(rows) + store.DumpRows(rows) } //nolint:unused -func (s *Store) DumpRows(rows *sql.Rows) { +func (store *Store) DumpRows(rows *sql.Rows) { data, err := xsql.Pretty(rows) if err != nil { panic(err) diff --git a/internal/storage/ledger/errors.go b/internal/storage/ledger/errors.go deleted file mode 100644 index e8a53ec7f..000000000 --- a/internal/storage/ledger/errors.go +++ /dev/null @@ -1,11 +0,0 @@ -package ledger - -import ( - "errors" -) - -var ( - ErrBucketAlreadyExists = errors.New("bucket already exists") - ErrStoreAlreadyExists = errors.New("store already exists") - ErrStoreNotFound = errors.New("store not found") -) diff --git a/internal/storage/ledger/legacy/accounts.go b/internal/storage/ledger/legacy/accounts.go index aa9004697..ccadb0c77 100644 --- a/internal/storage/ledger/legacy/accounts.go +++ b/internal/storage/ledger/legacy/accounts.go @@ -14,7 +14,7 @@ import ( "github.com/uptrace/bun" ) -func (store *Store) buildAccountQuery(q ledgercontroller.PITFilterWithVolumes, query *bun.SelectQuery) *bun.SelectQuery { +func (store *Store) buildAccountQuery(q PITFilterWithVolumes, query *bun.SelectQuery) *bun.SelectQuery { query = query. Column("accounts.address", "accounts.first_usage"). @@ -55,7 +55,7 @@ func (store *Store) buildAccountQuery(q ledgercontroller.PITFilterWithVolumes, q return query } -func (store *Store) accountQueryContext(qb query.Builder, q ledgercontroller.ListAccountsQuery) (string, []any, error) { +func (store *Store) accountQueryContext(qb query.Builder, q ListAccountsQuery) (string, []any, error) { metadataRegex := regexp.MustCompile(`metadata\[(.+)]`) balanceRegex := regexp.MustCompile(`balance\[(.*)]`) @@ -134,7 +134,7 @@ func (store *Store) accountQueryContext(qb query.Builder, q ledgercontroller.Lis })) } -func (store *Store) buildAccountListQuery(selectQuery *bun.SelectQuery, q ledgercontroller.ListAccountsQuery, where string, args []any) *bun.SelectQuery { +func (store *Store) buildAccountListQuery(selectQuery *bun.SelectQuery, q ListAccountsQuery, where string, args []any) *bun.SelectQuery { selectQuery = store.buildAccountQuery(q.Options.Options, selectQuery) if where != "" { @@ -144,7 +144,7 @@ func (store *Store) buildAccountListQuery(selectQuery *bun.SelectQuery, q ledger return selectQuery } -func (store *Store) GetAccountsWithVolumes(ctx context.Context, q ledgercontroller.ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { +func (store *Store) GetAccountsWithVolumes(ctx context.Context, q ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { var ( where string args []any @@ -157,15 +157,15 @@ func (store *Store) GetAccountsWithVolumes(ctx context.Context, q ledgercontroll } } - return paginateWithOffset[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes], ledger.Account](store, ctx, - (*bunpaginate.OffsetPaginatedQuery[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes]])(&q), + return paginateWithOffset[ledgercontroller.PaginatedQueryOptions[PITFilterWithVolumes], ledger.Account](store, ctx, + (*bunpaginate.OffsetPaginatedQuery[ledgercontroller.PaginatedQueryOptions[PITFilterWithVolumes]])(&q), func(query *bun.SelectQuery) *bun.SelectQuery { return store.buildAccountListQuery(query, q, where, args) }, ) } -func (store *Store) GetAccountWithVolumes(ctx context.Context, q ledgercontroller.GetAccountQuery) (*ledger.Account, error) { +func (store *Store) GetAccountWithVolumes(ctx context.Context, q GetAccountQuery) (*ledger.Account, error) { account, err := fetch[*ledger.Account](store, true, ctx, func(query *bun.SelectQuery) *bun.SelectQuery { query = store.buildAccountQuery(q.PITFilterWithVolumes, query). Where("accounts.address = ?", q.Addr). @@ -179,7 +179,7 @@ func (store *Store) GetAccountWithVolumes(ctx context.Context, q ledgercontrolle return account, nil } -func (store *Store) CountAccounts(ctx context.Context, q ledgercontroller.ListAccountsQuery) (int, error) { +func (store *Store) CountAccounts(ctx context.Context, q ListAccountsQuery) (int, error) { var ( where string args []any diff --git a/internal/storage/ledger/legacy/accounts_test.go b/internal/storage/ledger/legacy/accounts_test.go index 653bcdf7d..62201eaa8 100644 --- a/internal/storage/ledger/legacy/accounts_test.go +++ b/internal/storage/ledger/legacy/accounts_test.go @@ -16,6 +16,7 @@ import ( "github.com/formancehq/go-libs/v2/metadata" "github.com/formancehq/go-libs/v2/query" ledger "github.com/formancehq/ledger/internal" + ledgerstore "github.com/formancehq/ledger/internal/storage/ledger/legacy" "github.com/stretchr/testify/require" ) @@ -67,57 +68,16 @@ func TestGetAccounts(t *testing.T) { WithInsertedAt(now.Add(200*time.Millisecond)))) require.NoError(t, err) - //require.NoError(t, store.InsertLogs(ctx, - // ledger.ChainLogs( - // ledger.NewTransactionLog( - // ledger.NewTransaction(). - // WithPostings(ledger.NewPosting("world", "account:1", "USD", big.NewInt(100))). - // WithDate(now), - // map[string]metadata.Metadata{ - // "account:1": { - // "category": "4", - // }, - // }, - // ).WithDate(now), - // ledger.NewSetMetadataOnAccountLog(time.Now(), "account:1", metadata.Metadata{"category": "1"}).WithDate(now.Add(time.Minute)), - // ledger.NewSetMetadataOnAccountLog(time.Now(), "account:2", metadata.Metadata{"category": "2"}).WithDate(now.Add(2*time.Minute)), - // ledger.NewSetMetadataOnAccountLog(time.Now(), "account:3", metadata.Metadata{"category": "3"}).WithDate(now.Add(3*time.Minute)), - // ledger.NewSetMetadataOnAccountLog(time.Now(), "orders:1", metadata.Metadata{"foo": "bar"}).WithDate(now.Add(3*time.Minute)), - // ledger.NewSetMetadataOnAccountLog(time.Now(), "orders:2", metadata.Metadata{"foo": "bar"}).WithDate(now.Add(3*time.Minute)), - // ledger.NewTransactionLog( - // ledger.NewTransaction(). - // WithPostings(ledger.NewPosting("world", "account:1", "USD", big.NewInt(100))). - // WithIDUint64(1). - // WithDate(now.Add(4*time.Minute)), - // map[string]metadata.Metadata{}, - // ).WithDate(now.Add(100*time.Millisecond)), - // ledger.NewTransactionLog( - // ledger.NewTransaction(). - // WithPostings(ledger.NewPosting("account:1", "bank", "USD", big.NewInt(50))). - // WithDate(now.Add(3*time.Minute)). - // WithIDUint64(2), - // map[string]metadata.Metadata{}, - // ).WithDate(now.Add(200*time.Millisecond)), - // ledger.NewTransactionLog( - // ledger.NewTransaction(). - // WithPostings(ledger.NewPosting("world", "account:1", "USD", big.NewInt(0))). - // WithDate(now.Add(-time.Minute)). - // WithIDUint64(3), - // map[string]metadata.Metadata{}, - // ).WithDate(now.Add(200*time.Millisecond)), - // )..., - //)) - t.Run("list all", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}))) + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}))) require.NoError(t, err) require.Len(t, accounts.Data, 7) }) t.Run("list using metadata", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("metadata[category]", "1")), )) require.NoError(t, err) @@ -126,8 +86,8 @@ func TestGetAccounts(t *testing.T) { t.Run("list before date", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{ + PITFilter: ledgerstore.PITFilter{ PIT: &now, }, }))) @@ -138,7 +98,7 @@ func TestGetAccounts(t *testing.T) { t.Run("list with volumes", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{ ExpandVolumes: true, }).WithQueryBuilder(query.Match("address", "account:1")))) require.NoError(t, err) @@ -151,8 +111,8 @@ func TestGetAccounts(t *testing.T) { t.Run("list with volumes using PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{ + PITFilter: ledgerstore.PITFilter{ PIT: &now, }, ExpandVolumes: true, @@ -166,7 +126,7 @@ func TestGetAccounts(t *testing.T) { t.Run("list with effective volumes", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{ ExpandEffectiveVolumes: true, }).WithQueryBuilder(query.Match("address", "account:1")))) require.NoError(t, err) @@ -178,8 +138,8 @@ func TestGetAccounts(t *testing.T) { t.Run("list with effective volumes using PIT", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{ + PITFilter: ledgerstore.PITFilter{ PIT: &now, }, ExpandEffectiveVolumes: true, @@ -193,7 +153,7 @@ func TestGetAccounts(t *testing.T) { t.Run("list using filter on address", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("address", "account:")), )) require.NoError(t, err) @@ -201,7 +161,7 @@ func TestGetAccounts(t *testing.T) { }) t.Run("list using filter on multiple address", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder( query.Or( query.Match("address", "account:1"), @@ -214,13 +174,13 @@ func TestGetAccounts(t *testing.T) { }) t.Run("list using filter on balances", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Lt("balance[USD]", 0)), )) require.NoError(t, err) require.Len(t, accounts.Data, 1) // world - accounts, err = store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + accounts, err = store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Gt("balance[USD]", 0)), )) require.NoError(t, err) @@ -231,13 +191,13 @@ func TestGetAccounts(t *testing.T) { t.Run("list using filter on exists metadata", func(t *testing.T) { t.Parallel() - accounts, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + accounts, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Exists("metadata", "foo")), )) require.NoError(t, err) require.Len(t, accounts.Data, 2) - accounts, err = store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + accounts, err = store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Exists("metadata", "category")), )) require.NoError(t, err) @@ -246,7 +206,7 @@ func TestGetAccounts(t *testing.T) { t.Run("list using filter invalid field", func(t *testing.T) { t.Parallel() - _, err := store.GetAccountsWithVolumes(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + _, err := store.GetAccountsWithVolumes(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Lt("invalid", 0)), )) require.Error(t, err) @@ -278,7 +238,7 @@ func TestGetAccount(t *testing.T) { t.Run("find account", func(t *testing.T) { t.Parallel() - account, err := store.GetAccountWithVolumes(ctx, ledgercontroller.NewGetAccountQuery("multi")) + account, err := store.GetAccountWithVolumes(ctx, ledgerstore.NewGetAccountQuery("multi")) require.NoError(t, err) require.Equal(t, ledger.Account{ Address: "multi", @@ -288,7 +248,7 @@ func TestGetAccount(t *testing.T) { FirstUsage: now.Add(-time.Minute), }, *account) - account, err = store.GetAccountWithVolumes(ctx, ledgercontroller.NewGetAccountQuery("world")) + account, err = store.GetAccountWithVolumes(ctx, ledgerstore.NewGetAccountQuery("world")) require.NoError(t, err) require.Equal(t, ledger.Account{ Address: "world", @@ -299,7 +259,7 @@ func TestGetAccount(t *testing.T) { t.Run("find account in past", func(t *testing.T) { t.Parallel() - account, err := store.GetAccountWithVolumes(ctx, ledgercontroller.NewGetAccountQuery("multi").WithPIT(now.Add(-30*time.Second))) + account, err := store.GetAccountWithVolumes(ctx, ledgerstore.NewGetAccountQuery("multi").WithPIT(now.Add(-30*time.Second))) require.NoError(t, err) require.Equal(t, ledger.Account{ Address: "multi", @@ -310,7 +270,7 @@ func TestGetAccount(t *testing.T) { t.Run("find account with volumes", func(t *testing.T) { t.Parallel() - account, err := store.GetAccountWithVolumes(ctx, ledgercontroller.NewGetAccountQuery("multi"). + account, err := store.GetAccountWithVolumes(ctx, ledgerstore.NewGetAccountQuery("multi"). WithExpandVolumes()) require.NoError(t, err) require.Equal(t, ledger.Account{ @@ -327,7 +287,7 @@ func TestGetAccount(t *testing.T) { t.Run("find account with effective volumes", func(t *testing.T) { t.Parallel() - account, err := store.GetAccountWithVolumes(ctx, ledgercontroller.NewGetAccountQuery("multi"). + account, err := store.GetAccountWithVolumes(ctx, ledgerstore.NewGetAccountQuery("multi"). WithExpandEffectiveVolumes()) require.NoError(t, err) require.Equal(t, ledger.Account{ @@ -345,7 +305,7 @@ func TestGetAccount(t *testing.T) { t.Run("find account using pit", func(t *testing.T) { t.Parallel() - account, err := store.GetAccountWithVolumes(ctx, ledgercontroller.NewGetAccountQuery("multi").WithPIT(now)) + account, err := store.GetAccountWithVolumes(ctx, ledgerstore.NewGetAccountQuery("multi").WithPIT(now)) require.NoError(t, err) require.Equal(t, ledger.Account{ Address: "multi", @@ -356,7 +316,7 @@ func TestGetAccount(t *testing.T) { t.Run("not existent account", func(t *testing.T) { t.Parallel() - _, err := store.GetAccountWithVolumes(ctx, ledgercontroller.NewGetAccountQuery("account_not_existing")) + _, err := store.GetAccountWithVolumes(ctx, ledgerstore.NewGetAccountQuery("account_not_existing")) require.Error(t, err) }) @@ -372,7 +332,7 @@ func TestCountAccounts(t *testing.T) { ))) require.NoError(t, err) - countAccounts, err := store.CountAccounts(ctx, ledgercontroller.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}))) + countAccounts, err := store.CountAccounts(ctx, ledgerstore.NewListAccountsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}))) require.NoError(t, err) require.EqualValues(t, 2, countAccounts) // world + central_bank } diff --git a/internal/storage/ledger/legacy/adapters.go b/internal/storage/ledger/legacy/adapters.go index e6ad4b72a..3ea005891 100644 --- a/internal/storage/ledger/legacy/adapters.go +++ b/internal/storage/ledger/legacy/adapters.go @@ -3,7 +3,6 @@ package legacy import ( "context" "database/sql" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/metadata" "github.com/formancehq/go-libs/v2/migrations" "github.com/formancehq/go-libs/v2/time" @@ -19,6 +18,27 @@ type DefaultStoreAdapter struct { isFullUpToDate bool } +// todo; handle compat with v1 +func (d *DefaultStoreAdapter) Accounts() ledgercontroller.PaginatedResource[ledger.Account, any, ledgercontroller.OffsetPaginatedQuery[any]] { + return d.newStore.Accounts() +} + +func (d *DefaultStoreAdapter) Logs() ledgercontroller.PaginatedResource[ledger.Log, any, ledgercontroller.ColumnPaginatedQuery[any]] { + return d.newStore.Logs() +} + +func (d *DefaultStoreAdapter) Transactions() ledgercontroller.PaginatedResource[ledger.Transaction, any, ledgercontroller.ColumnPaginatedQuery[any]] { + return d.newStore.Transactions() +} + +func (d *DefaultStoreAdapter) AggregatedBalances() ledgercontroller.Resource[ledger.AggregatedVolumes, ledgercontroller.GetAggregatedVolumesOptions] { + return d.newStore.AggregatedVolumes() +} + +func (d *DefaultStoreAdapter) Volumes() ledgercontroller.PaginatedResource[ledger.VolumesWithBalanceByAssetByAccount, ledgercontroller.GetVolumesOptions, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]] { + return d.newStore.Volumes() +} + func (d *DefaultStoreAdapter) GetDB() bun.IDB { return d.newStore.GetDB() } @@ -47,7 +67,7 @@ func (d *DefaultStoreAdapter) UpdateAccountsMetadata(ctx context.Context, m map[ return d.newStore.UpdateAccountsMetadata(ctx, m) } -func (d *DefaultStoreAdapter) UpsertAccounts(ctx context.Context, accounts ... *ledger.Account) error { +func (d *DefaultStoreAdapter) UpsertAccounts(ctx context.Context, accounts ...*ledger.Account) error { return d.newStore.UpsertAccounts(ctx, accounts...) } @@ -63,82 +83,10 @@ func (d *DefaultStoreAdapter) LockLedger(ctx context.Context) error { return d.newStore.LockLedger(ctx) } -func (d *DefaultStoreAdapter) ListLogs(ctx context.Context, q ledgercontroller.GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { - if !d.isFullUpToDate { - return d.legacyStore.GetLogs(ctx, q) - } - - return d.newStore.ListLogs(ctx, q) -} - func (d *DefaultStoreAdapter) ReadLogWithIdempotencyKey(ctx context.Context, ik string) (*ledger.Log, error) { return d.newStore.ReadLogWithIdempotencyKey(ctx, ik) } -func (d *DefaultStoreAdapter) ListTransactions(ctx context.Context, q ledgercontroller.ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { - if !d.isFullUpToDate { - return d.legacyStore.GetTransactions(ctx, q) - } - - return d.newStore.ListTransactions(ctx, q) -} - -func (d *DefaultStoreAdapter) CountTransactions(ctx context.Context, q ledgercontroller.ListTransactionsQuery) (int, error) { - if !d.isFullUpToDate { - return d.legacyStore.CountTransactions(ctx, q) - } - - return d.newStore.CountTransactions(ctx, q) -} - -func (d *DefaultStoreAdapter) GetTransaction(ctx context.Context, query ledgercontroller.GetTransactionQuery) (*ledger.Transaction, error) { - if !d.isFullUpToDate { - return d.legacyStore.GetTransactionWithVolumes(ctx, query) - } - - return d.newStore.GetTransaction(ctx, query) -} - -func (d *DefaultStoreAdapter) CountAccounts(ctx context.Context, q ledgercontroller.ListAccountsQuery) (int, error) { - if !d.isFullUpToDate { - return d.legacyStore.CountAccounts(ctx, q) - } - - return d.newStore.CountAccounts(ctx, q) -} - -func (d *DefaultStoreAdapter) ListAccounts(ctx context.Context, q ledgercontroller.ListAccountsQuery) (*bunpaginate.Cursor[ledger.Account], error) { - if !d.isFullUpToDate { - return d.legacyStore.GetAccountsWithVolumes(ctx, q) - } - - return d.newStore.ListAccounts(ctx, q) -} - -func (d *DefaultStoreAdapter) GetAccount(ctx context.Context, q ledgercontroller.GetAccountQuery) (*ledger.Account, error) { - if !d.isFullUpToDate { - return d.legacyStore.GetAccountWithVolumes(ctx, q) - } - - return d.newStore.GetAccount(ctx, q) -} - -func (d *DefaultStoreAdapter) GetAggregatedBalances(ctx context.Context, q ledgercontroller.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { - if !d.isFullUpToDate { - return d.legacyStore.GetAggregatedBalances(ctx, q) - } - - return d.newStore.GetAggregatedBalances(ctx, q) -} - -func (d *DefaultStoreAdapter) GetVolumesWithBalances(ctx context.Context, q ledgercontroller.GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { - if !d.isFullUpToDate { - return d.legacyStore.GetVolumesWithBalances(ctx, q) - } - - return d.newStore.GetVolumesWithBalances(ctx, q) -} - func (d *DefaultStoreAdapter) IsUpToDate(ctx context.Context) (bool, error) { return d.newStore.HasMinimalVersion(ctx) } diff --git a/internal/storage/ledger/legacy/balances.go b/internal/storage/ledger/legacy/balances.go index 5266fdb6d..7a3b02cbd 100644 --- a/internal/storage/ledger/legacy/balances.go +++ b/internal/storage/ledger/legacy/balances.go @@ -7,11 +7,10 @@ import ( "github.com/formancehq/go-libs/v2/platform/postgres" "github.com/formancehq/go-libs/v2/query" ledger "github.com/formancehq/ledger/internal" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/uptrace/bun" ) -func (store *Store) GetAggregatedBalances(ctx context.Context, q ledgercontroller.GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { +func (store *Store) GetAggregatedBalances(ctx context.Context, q GetAggregatedBalanceQuery) (ledger.BalancesByAssets, error) { var ( needMetadata bool diff --git a/internal/storage/ledger/legacy/balances_test.go b/internal/storage/ledger/legacy/balances_test.go index d6c185946..e2adaac9d 100644 --- a/internal/storage/ledger/legacy/balances_test.go +++ b/internal/storage/ledger/legacy/balances_test.go @@ -3,7 +3,7 @@ package legacy_test import ( - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + ledgerstore "github.com/formancehq/ledger/internal/storage/ledger/legacy" "github.com/google/go-cmp/cmp" "math/big" "testing" @@ -74,7 +74,7 @@ func TestGetBalancesAggregated(t *testing.T) { t.Run("aggregate on all", func(t *testing.T) { t.Parallel() - cursor, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, nil, false)) + cursor, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{}, nil, false)) require.NoError(t, err) RequireEqual(t, ledger.BalancesByAssets{ "USD": big.NewInt(0), @@ -83,7 +83,7 @@ func TestGetBalancesAggregated(t *testing.T) { }) t.Run("filter on address", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, + ret, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{}, query.Match("address", "users:"), false)) require.NoError(t, err) require.Equal(t, ledger.BalancesByAssets{ @@ -95,7 +95,7 @@ func TestGetBalancesAggregated(t *testing.T) { }) t.Run("using pit on effective date", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{ + ret, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{ PIT: pointer.For(now.Add(-time.Second)), }, query.Match("address", "users:"), false)) require.NoError(t, err) @@ -108,7 +108,7 @@ func TestGetBalancesAggregated(t *testing.T) { }) t.Run("using pit on insertion date", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{ + ret, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{ PIT: pointer.For(now), }, query.Match("address", "users:"), true)) require.NoError(t, err) @@ -122,7 +122,7 @@ func TestGetBalancesAggregated(t *testing.T) { t.Run("using a metadata and pit", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{ + ret, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{ PIT: pointer.For(now.Add(time.Minute)), }, query.Match("metadata[category]", "premium"), false)) require.NoError(t, err) @@ -135,7 +135,7 @@ func TestGetBalancesAggregated(t *testing.T) { }) t.Run("using a metadata without pit", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, + ret, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{}, query.Match("metadata[category]", "premium"), false)) require.NoError(t, err) require.Equal(t, ledger.BalancesByAssets{ @@ -144,7 +144,7 @@ func TestGetBalancesAggregated(t *testing.T) { }) t.Run("when no matching", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, + ret, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{}, query.Match("metadata[category]", "guest"), false)) require.NoError(t, err) require.Equal(t, ledger.BalancesByAssets{}, ret) @@ -152,7 +152,7 @@ func TestGetBalancesAggregated(t *testing.T) { t.Run("using a filter exist on metadata", func(t *testing.T) { t.Parallel() - ret, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{}, query.Exists("metadata", "category"), false)) + ret, err := store.GetAggregatedBalances(ctx, ledgerstore.NewGetAggregatedBalancesQuery(ledgerstore.PITFilter{}, query.Exists("metadata", "category"), false)) require.NoError(t, err) require.Equal(t, ledger.BalancesByAssets{ "USD": big.NewInt(0).Add( diff --git a/internal/storage/ledger/legacy/logs.go b/internal/storage/ledger/legacy/logs.go index 069f4fd94..a022aa789 100644 --- a/internal/storage/ledger/legacy/logs.go +++ b/internal/storage/ledger/legacy/logs.go @@ -35,7 +35,7 @@ func (store *Store) logsQueryBuilder(q ledgercontroller.PaginatedQueryOptions[an } } -func (store *Store) GetLogs(ctx context.Context, q ledgercontroller.GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { +func (store *Store) GetLogs(ctx context.Context, q GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { logs, err := paginateWithColumn[ledgercontroller.PaginatedQueryOptions[any], ledgerstore.Log](store, ctx, (*bunpaginate.ColumnPaginatedQuery[ledgercontroller.PaginatedQueryOptions[any]])(&q), store.logsQueryBuilder(q.Options), diff --git a/internal/storage/ledger/legacy/logs_test.go b/internal/storage/ledger/legacy/logs_test.go index 1f4f297f5..6e6e298f9 100644 --- a/internal/storage/ledger/legacy/logs_test.go +++ b/internal/storage/ledger/legacy/logs_test.go @@ -4,6 +4,7 @@ package legacy_test import ( ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + ledgerstore "github.com/formancehq/ledger/internal/storage/ledger/legacy" "testing" "github.com/formancehq/go-libs/v2/bun/bunpaginate" @@ -33,20 +34,20 @@ func TestLogsList(t *testing.T) { require.NoError(t, err) } - cursor, err := store.GetLogs(ctx, ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil))) + cursor, err := store.GetLogs(ctx, ledgerstore.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil))) require.NoError(t, err) require.Equal(t, bunpaginate.QueryDefaultPageSize, cursor.PageSize) require.Equal(t, 3, len(cursor.Data)) require.EqualValues(t, 3, cursor.Data[0].ID) - cursor, err = store.GetLogs(ctx, ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil).WithPageSize(1))) + cursor, err = store.GetLogs(ctx, ledgerstore.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil).WithPageSize(1))) require.NoError(t, err) // Should get only the first log. require.Equal(t, 1, cursor.PageSize) require.EqualValues(t, 3, cursor.Data[0].ID) - cursor, err = store.GetLogs(ctx, ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil). + cursor, err = store.GetLogs(ctx, ledgerstore.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil). WithQueryBuilder(query.And( query.Gte("date", now.Add(-2*time.Hour)), query.Lt("date", now.Add(-time.Hour)), diff --git a/internal/storage/ledger/legacy/queries.go b/internal/storage/ledger/legacy/queries.go new file mode 100644 index 000000000..ade76c7aa --- /dev/null +++ b/internal/storage/ledger/legacy/queries.go @@ -0,0 +1,159 @@ +package legacy + +import ( + "github.com/formancehq/go-libs/v2/bun/bunpaginate" + "github.com/formancehq/go-libs/v2/pointer" + "github.com/formancehq/go-libs/v2/query" + "github.com/formancehq/go-libs/v2/time" + "github.com/formancehq/ledger/internal/controller/ledger" +) + +type PITFilter struct { + PIT *time.Time `json:"pit"` + OOT *time.Time `json:"oot"` +} + +type PITFilterWithVolumes struct { + PITFilter + ExpandVolumes bool `json:"volumes"` + ExpandEffectiveVolumes bool `json:"effectiveVolumes"` +} + +type FiltersForVolumes struct { + PITFilter + UseInsertionDate bool + GroupLvl int +} + +func NewGetVolumesWithBalancesQuery(opts ledger.PaginatedQueryOptions[FiltersForVolumes]) GetVolumesWithBalancesQuery { + return GetVolumesWithBalancesQuery{ + PageSize: opts.PageSize, + Order: bunpaginate.OrderAsc, + Options: opts, + } +} + +type ListTransactionsQuery bunpaginate.ColumnPaginatedQuery[ledger.PaginatedQueryOptions[PITFilterWithVolumes]] + +func (q ListTransactionsQuery) WithColumn(column string) ListTransactionsQuery { + ret := pointer.For((bunpaginate.ColumnPaginatedQuery[ledger.PaginatedQueryOptions[PITFilterWithVolumes]])(q)) + ret = ret.WithColumn(column) + + return ListTransactionsQuery(*ret) +} + +func NewListTransactionsQuery(options ledger.PaginatedQueryOptions[PITFilterWithVolumes]) ListTransactionsQuery { + return ListTransactionsQuery{ + PageSize: options.PageSize, + Column: "id", + Order: bunpaginate.OrderDesc, + Options: options, + } +} + +type GetTransactionQuery struct { + PITFilterWithVolumes + ID int +} + +func (q GetTransactionQuery) WithExpandVolumes() GetTransactionQuery { + q.ExpandVolumes = true + + return q +} + +func (q GetTransactionQuery) WithExpandEffectiveVolumes() GetTransactionQuery { + q.ExpandEffectiveVolumes = true + + return q +} + +func NewGetTransactionQuery(id int) GetTransactionQuery { + return GetTransactionQuery{ + PITFilterWithVolumes: PITFilterWithVolumes{}, + ID: id, + } +} + +type ListAccountsQuery bunpaginate.OffsetPaginatedQuery[ledger.PaginatedQueryOptions[PITFilterWithVolumes]] + +func (q ListAccountsQuery) WithExpandVolumes() ListAccountsQuery { + q.Options.Options.ExpandVolumes = true + + return q +} + +func (q ListAccountsQuery) WithExpandEffectiveVolumes() ListAccountsQuery { + q.Options.Options.ExpandEffectiveVolumes = true + + return q +} + +func NewListAccountsQuery(opts ledger.PaginatedQueryOptions[PITFilterWithVolumes]) ListAccountsQuery { + return ListAccountsQuery{ + PageSize: opts.PageSize, + Order: bunpaginate.OrderAsc, + Options: opts, + } +} + +type GetAccountQuery struct { + PITFilterWithVolumes + Addr string +} + +func (q GetAccountQuery) WithPIT(pit time.Time) GetAccountQuery { + q.PIT = &pit + + return q +} + +func (q GetAccountQuery) WithExpandVolumes() GetAccountQuery { + q.ExpandVolumes = true + + return q +} + +func (q GetAccountQuery) WithExpandEffectiveVolumes() GetAccountQuery { + q.ExpandEffectiveVolumes = true + + return q +} + +func NewGetAccountQuery(addr string) GetAccountQuery { + return GetAccountQuery{ + Addr: addr, + } +} + +type GetAggregatedBalanceQuery struct { + PITFilter + QueryBuilder query.Builder + UseInsertionDate bool +} + +func NewGetAggregatedBalancesQuery(filter PITFilter, qb query.Builder, useInsertionDate bool) GetAggregatedBalanceQuery { + return GetAggregatedBalanceQuery{ + PITFilter: filter, + QueryBuilder: qb, + UseInsertionDate: useInsertionDate, + } +} + +type GetVolumesWithBalancesQuery bunpaginate.OffsetPaginatedQuery[ledger.PaginatedQueryOptions[FiltersForVolumes]] + +type GetLogsQuery bunpaginate.ColumnPaginatedQuery[ledger.PaginatedQueryOptions[any]] + +func (q GetLogsQuery) WithOrder(order bunpaginate.Order) GetLogsQuery { + q.Order = order + return q +} + +func NewListLogsQuery(options ledger.PaginatedQueryOptions[any]) GetLogsQuery { + return GetLogsQuery{ + PageSize: options.PageSize, + Column: "id", + Order: bunpaginate.OrderDesc, + Options: options, + } +} diff --git a/internal/storage/ledger/legacy/transactions.go b/internal/storage/ledger/legacy/transactions.go index a7dfa99ae..b9bd8399a 100644 --- a/internal/storage/ledger/legacy/transactions.go +++ b/internal/storage/ledger/legacy/transactions.go @@ -20,7 +20,7 @@ var ( metadataRegex = regexp.MustCompile(`metadata\[(.+)]`) ) -func (store *Store) buildTransactionQuery(p ledgercontroller.PITFilterWithVolumes, query *bun.SelectQuery) *bun.SelectQuery { +func (store *Store) buildTransactionQuery(p PITFilterWithVolumes, query *bun.SelectQuery) *bun.SelectQuery { selectMetadata := query.NewSelect(). ModelTableExpr(store.GetPrefixedRelationName("transactions_metadata")). @@ -64,7 +64,7 @@ func (store *Store) buildTransactionQuery(p ledgercontroller.PITFilterWithVolume return query } -func (store *Store) transactionQueryContext(qb query.Builder, q ledgercontroller.ListTransactionsQuery) (string, []any, error) { +func (store *Store) transactionQueryContext(qb query.Builder, q ListTransactionsQuery) (string, []any, error) { return qb.Build(query.ContextFn(func(key, operator string, value any) (string, []any, error) { switch { @@ -144,7 +144,7 @@ func (store *Store) transactionQueryContext(qb query.Builder, q ledgercontroller })) } -func (store *Store) buildTransactionListQuery(selectQuery *bun.SelectQuery, q ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes], where string, args []any) *bun.SelectQuery { +func (store *Store) buildTransactionListQuery(selectQuery *bun.SelectQuery, q ledgercontroller.PaginatedQueryOptions[PITFilterWithVolumes], where string, args []any) *bun.SelectQuery { selectQuery = store.buildTransactionQuery(q.Options, selectQuery) if where != "" { @@ -154,7 +154,7 @@ func (store *Store) buildTransactionListQuery(selectQuery *bun.SelectQuery, q le return selectQuery } -func (store *Store) GetTransactions(ctx context.Context, q ledgercontroller.ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { +func (store *Store) GetTransactions(ctx context.Context, q ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { var ( where string @@ -168,15 +168,15 @@ func (store *Store) GetTransactions(ctx context.Context, q ledgercontroller.List } } - return paginateWithColumn[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes], ledger.Transaction](store, ctx, - (*bunpaginate.ColumnPaginatedQuery[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes]])(&q), + return paginateWithColumn[ledgercontroller.PaginatedQueryOptions[PITFilterWithVolumes], ledger.Transaction](store, ctx, + (*bunpaginate.ColumnPaginatedQuery[ledgercontroller.PaginatedQueryOptions[PITFilterWithVolumes]])(&q), func(query *bun.SelectQuery) *bun.SelectQuery { return store.buildTransactionListQuery(query, q.Options, where, args) }, ) } -func (store *Store) CountTransactions(ctx context.Context, q ledgercontroller.ListTransactionsQuery) (int, error) { +func (store *Store) CountTransactions(ctx context.Context, q ListTransactionsQuery) (int, error) { var ( where string @@ -196,7 +196,7 @@ func (store *Store) CountTransactions(ctx context.Context, q ledgercontroller.Li }) } -func (store *Store) GetTransactionWithVolumes(ctx context.Context, filter ledgercontroller.GetTransactionQuery) (*ledger.Transaction, error) { +func (store *Store) GetTransactionWithVolumes(ctx context.Context, filter GetTransactionQuery) (*ledger.Transaction, error) { return fetch[*ledger.Transaction](store, true, ctx, func(query *bun.SelectQuery) *bun.SelectQuery { return store.buildTransactionQuery(filter.PITFilterWithVolumes, query). diff --git a/internal/storage/ledger/legacy/transactions_test.go b/internal/storage/ledger/legacy/transactions_test.go index 090a6be47..23b778f5a 100644 --- a/internal/storage/ledger/legacy/transactions_test.go +++ b/internal/storage/ledger/legacy/transactions_test.go @@ -6,6 +6,7 @@ import ( "context" "fmt" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + ledgerstore "github.com/formancehq/ledger/internal/storage/ledger/legacy" "math/big" "testing" @@ -46,7 +47,7 @@ func TestGetTransactionWithVolumes(t *testing.T) { err = store.newStore.CommitTransaction(ctx, &tx2) require.NoError(t, err) - tx, err := store.GetTransactionWithVolumes(ctx, ledgercontroller.NewGetTransactionQuery(tx1.ID). + tx, err := store.GetTransactionWithVolumes(ctx, ledgerstore.NewGetTransactionQuery(tx1.ID). WithExpandVolumes(). WithExpandEffectiveVolumes()) require.NoError(t, err) @@ -68,7 +69,7 @@ func TestGetTransactionWithVolumes(t *testing.T) { }, }, tx.PostCommitVolumes) - tx, err = store.GetTransactionWithVolumes(ctx, ledgercontroller.NewGetTransactionQuery(tx2.ID). + tx, err = store.GetTransactionWithVolumes(ctx, ledgerstore.NewGetTransactionQuery(tx2.ID). WithExpandVolumes(). WithExpandEffectiveVolumes()) require.NoError(t, err) @@ -104,7 +105,7 @@ func TestCountTransactions(t *testing.T) { require.NoError(t, err) } - count, err := store.CountTransactions(context.Background(), ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}))) + count, err := store.CountTransactions(context.Background(), ledgerstore.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}))) require.NoError(t, err, "counting transactions should not fail") require.Equal(t, 3, count, "count should be equal") } @@ -158,7 +159,7 @@ func TestGetTransactions(t *testing.T) { // refresh tx3 // we can't take the result of the call on RevertTransaction nor UpdateTransactionMetadata as the result does not contains pc(e)v tx3 := func() ledger.Transaction { - tx3, err := store.newStore.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx3BeforeRevert.ID). + tx3, err := store.Store.GetTransactionWithVolumes(ctx, ledgerstore.NewGetTransactionQuery(tx3BeforeRevert.ID). WithExpandVolumes(). WithExpandEffectiveVolumes()) require.NoError(t, err) @@ -175,44 +176,44 @@ func TestGetTransactions(t *testing.T) { type testCase struct { name string - query ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + query ledgercontroller.PaginatedQueryOptions[ledgerstore.PITFilterWithVolumes] expected []ledger.Transaction expectError error } testCases := []testCase{ { name: "nominal", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}), + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}), expected: []ledger.Transaction{tx5, tx4, tx3, tx2, tx1}, }, { name: "address filter", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("account", "bob")), expected: []ledger.Transaction{tx2}, }, { name: "address filter using segments matching two addresses by individual segments", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("account", "users:amazon")), expected: []ledger.Transaction{}, }, { name: "address filter using segment", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("account", "users:")), expected: []ledger.Transaction{tx5, tx4, tx3}, }, { name: "filter using metadata", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("metadata[category]", "2")), expected: []ledger.Transaction{tx2}, }, { name: "using point in time", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{ + PITFilter: ledgerstore.PITFilter{ PIT: pointer.For(now.Add(-time.Hour)), }, }), @@ -220,20 +221,20 @@ func TestGetTransactions(t *testing.T) { }, { name: "reverted transactions", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("reverted", true)), expected: []ledger.Transaction{tx3}, }, { name: "filter using exists metadata", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Exists("metadata", "category")), expected: []ledger.Transaction{tx3, tx2, tx1}, }, { name: "filter using metadata and pit", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{ + PITFilter: ledgerstore.PITFilter{ PIT: pointer.For(tx3.Timestamp), }, }). @@ -242,13 +243,13 @@ func TestGetTransactions(t *testing.T) { }, { name: "filter using not exists metadata", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Not(query.Exists("metadata", "category"))), expected: []ledger.Transaction{tx5, tx4}, }, { name: "filter using timestamp", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). + query: ledgercontroller.NewPaginatedQueryOptions(ledgerstore.PITFilterWithVolumes{}). WithQueryBuilder(query.Match("timestamp", tx5.Timestamp.Format(time.RFC3339Nano))), expected: []ledger.Transaction{tx5, tx4}, }, @@ -262,7 +263,7 @@ func TestGetTransactions(t *testing.T) { tc.query.Options.ExpandVolumes = true tc.query.Options.ExpandEffectiveVolumes = true - cursor, err := store.GetTransactions(ctx, ledgercontroller.NewListTransactionsQuery(tc.query)) + cursor, err := store.GetTransactions(ctx, ledgerstore.NewListTransactionsQuery(tc.query)) if tc.expectError != nil { require.True(t, errors.Is(err, tc.expectError)) } else { @@ -270,7 +271,7 @@ func TestGetTransactions(t *testing.T) { require.Len(t, cursor.Data, len(tc.expected)) RequireEqual(t, tc.expected, cursor.Data) - count, err := store.CountTransactions(ctx, ledgercontroller.NewListTransactionsQuery(tc.query)) + count, err := store.CountTransactions(ctx, ledgerstore.NewListTransactionsQuery(tc.query)) require.NoError(t, err) require.EqualValues(t, len(tc.expected), count) diff --git a/internal/storage/ledger/legacy/volumes.go b/internal/storage/ledger/legacy/volumes.go index 1e079ae0f..c427227c6 100644 --- a/internal/storage/ledger/legacy/volumes.go +++ b/internal/storage/ledger/legacy/volumes.go @@ -12,7 +12,7 @@ import ( "github.com/uptrace/bun" ) -func (store *Store) volumesQueryContext(q ledgercontroller.GetVolumesWithBalancesQuery) (string, []any, bool, error) { +func (store *Store) volumesQueryContext(q GetVolumesWithBalancesQuery) (string, []any, bool, error) { metadataRegex := regexp.MustCompile(`metadata\[(.+)]`) balanceRegex := regexp.MustCompile(`balance\[(.*)]`) @@ -90,7 +90,7 @@ func (store *Store) volumesQueryContext(q ledgercontroller.GetVolumesWithBalance } -func (store *Store) buildVolumesWithBalancesQuery(query *bun.SelectQuery, q ledgercontroller.GetVolumesWithBalancesQuery, where string, args []any, useMetadata bool) *bun.SelectQuery { +func (store *Store) buildVolumesWithBalancesQuery(query *bun.SelectQuery, q GetVolumesWithBalancesQuery, where string, args []any, useMetadata bool) *bun.SelectQuery { filtersForVolumes := q.Options.Options dateFilterColumn := "effective_date" @@ -165,7 +165,7 @@ func (store *Store) buildVolumesWithBalancesQuery(query *bun.SelectQuery, q ledg return globalQuery } -func (store *Store) GetVolumesWithBalances(ctx context.Context, q ledgercontroller.GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { +func (store *Store) GetVolumesWithBalances(ctx context.Context, q GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { var ( where string args []any @@ -179,8 +179,8 @@ func (store *Store) GetVolumesWithBalances(ctx context.Context, q ledgercontroll } } - return paginateWithOffsetWithoutModel[ledgercontroller.PaginatedQueryOptions[ledgercontroller.FiltersForVolumes], ledger.VolumesWithBalanceByAssetByAccount]( - store, ctx, (*bunpaginate.OffsetPaginatedQuery[ledgercontroller.PaginatedQueryOptions[ledgercontroller.FiltersForVolumes]])(&q), + return paginateWithOffsetWithoutModel[ledgercontroller.PaginatedQueryOptions[FiltersForVolumes], ledger.VolumesWithBalanceByAssetByAccount]( + store, ctx, (*bunpaginate.OffsetPaginatedQuery[ledgercontroller.PaginatedQueryOptions[FiltersForVolumes]])(&q), func(query *bun.SelectQuery) *bun.SelectQuery { return store.buildVolumesWithBalancesQuery(query, q, where, args, useMetadata) }, diff --git a/internal/storage/ledger/legacy/volumes_test.go b/internal/storage/ledger/legacy/volumes_test.go index c5ca7a0a8..4f5553167 100644 --- a/internal/storage/ledger/legacy/volumes_test.go +++ b/internal/storage/ledger/legacy/volumes_test.go @@ -5,6 +5,7 @@ package legacy_test import ( "github.com/formancehq/go-libs/v2/pointer" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + ledgerstore "github.com/formancehq/ledger/internal/storage/ledger/legacy" "math/big" "testing" @@ -100,7 +101,7 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{UseInsertionDate: true}))) + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.FiltersForVolumes{UseInsertionDate: true}))) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -108,7 +109,7 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{UseInsertionDate: false}))) + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions(ledgerstore.FiltersForVolumes{UseInsertionDate: false}))) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -116,9 +117,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &previousPIT, OOT: nil}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &previousPIT, OOT: nil}, UseInsertionDate: true, }))) @@ -137,9 +138,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with futur pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: nil}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &futurPIT, OOT: nil}, UseInsertionDate: true, }))) require.NoError(t, err) @@ -149,9 +150,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &previousOOT}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: nil, OOT: &previousOOT}, UseInsertionDate: true, }))) require.NoError(t, err) @@ -161,9 +162,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with future oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &futurOOT}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: nil, OOT: &futurOOT}, UseInsertionDate: true, }))) @@ -182,9 +183,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with previous pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &previousPIT, OOT: nil}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &previousPIT, OOT: nil}, UseInsertionDate: false, }))) @@ -203,9 +204,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with futur pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: nil}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &futurPIT, OOT: nil}, UseInsertionDate: false, }))) require.NoError(t, err) @@ -215,9 +216,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with previous oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &previousOOT}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: nil, OOT: &previousOOT}, UseInsertionDate: false, }))) require.NoError(t, err) @@ -227,9 +228,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with futur oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &futurOOT}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: nil, OOT: &futurOOT}, UseInsertionDate: false, }))) @@ -248,9 +249,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with future PIT and now OOT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: &now}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &futurPIT, OOT: &now}, UseInsertionDate: true, }))) @@ -270,9 +271,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous OOT and now PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &now, OOT: &previousOOT}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &now, OOT: &previousOOT}, UseInsertionDate: true, }))) @@ -292,9 +293,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with future PIT and now OOT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: &now}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &futurPIT, OOT: &now}, UseInsertionDate: false, }))) @@ -313,9 +314,9 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous OOT and now PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &now, OOT: &previousOOT}, + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &now, OOT: &previousOOT}, UseInsertionDate: false, }))) @@ -337,10 +338,10 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( + ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &now, OOT: &previousOOT}, + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{PIT: &now, OOT: &previousOOT}, UseInsertionDate: false, }).WithQueryBuilder(query.Match("account", "account:1"))), ) @@ -363,9 +364,9 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( + ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{}).WithQueryBuilder(query.Match("metadata[foo]", "bar"))), + ledgerstore.FiltersForVolumes{}).WithQueryBuilder(query.Match("metadata[foo]", "bar"))), ) require.NoError(t, err) @@ -377,9 +378,9 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( + ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{}).WithQueryBuilder(query.Exists("metadata", "category"))), + ledgerstore.FiltersForVolumes{}).WithQueryBuilder(query.Exists("metadata", "category"))), ) require.NoError(t, err) @@ -390,9 +391,9 @@ func TestVolumesList(t *testing.T) { t.Parallel() volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( + ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{}).WithQueryBuilder(query.Exists("metadata", "foo"))), + ledgerstore.FiltersForVolumes{}).WithQueryBuilder(query.Exists("metadata", "foo"))), ) require.NoError(t, err) @@ -461,9 +462,9 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 0", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + ledgerstore.FiltersForVolumes{ UseInsertionDate: true, GroupLvl: 0, }).WithQueryBuilder(query.Match("account", "account::")))) @@ -474,9 +475,9 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + ledgerstore.FiltersForVolumes{ UseInsertionDate: true, GroupLvl: 1, }).WithQueryBuilder(query.Match("account", "account::")))) @@ -487,9 +488,9 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 2", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + ledgerstore.FiltersForVolumes{ UseInsertionDate: true, GroupLvl: 2, }).WithQueryBuilder(query.Match("account", "account::")))) @@ -500,9 +501,9 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 3", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + ledgerstore.FiltersForVolumes{ UseInsertionDate: true, GroupLvl: 3, }).WithQueryBuilder(query.Match("account", "account::")))) @@ -514,10 +515,10 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && PIT && OOT && effectiveDate", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{ PIT: &pit, OOT: &oot, }, @@ -548,10 +549,10 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && PIT && OOT && effectiveDate && Balance Filter 1", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{ PIT: &pit, OOT: &oot, }, @@ -575,10 +576,10 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && Balance Filter 2", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( + volumes, err := store.GetVolumesWithBalances(ctx, ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{}, + ledgerstore.FiltersForVolumes{ + PITFilter: ledgerstore.PITFilter{}, UseInsertionDate: true, GroupLvl: 2, }).WithQueryBuilder( @@ -620,9 +621,9 @@ func TestVolumesAggregate(t *testing.T) { t.Parallel() volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( + ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + ledgerstore.FiltersForVolumes{ GroupLvl: 1, }).WithQueryBuilder(query.And( query.Match("account", "account::"), @@ -638,11 +639,11 @@ func TestVolumesAggregate(t *testing.T) { t.Parallel() volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( + ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + ledgerstore.FiltersForVolumes{ GroupLvl: 1, - PITFilter: ledgercontroller.PITFilter{ + PITFilter: ledgerstore.PITFilter{ PIT: pointer.For(now.Add(time.Minute)), }, }).WithQueryBuilder(query.And( @@ -659,9 +660,9 @@ func TestVolumesAggregate(t *testing.T) { t.Parallel() volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( + ledgerstore.NewGetVolumesWithBalancesQuery( ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + ledgerstore.FiltersForVolumes{ GroupLvl: 1, }).WithQueryBuilder(query.And( query.Match("metadata[foo]", "bar"), diff --git a/internal/storage/ledger/logs.go b/internal/storage/ledger/logs.go index 2f6da528f..47d764d2e 100644 --- a/internal/storage/ledger/logs.go +++ b/internal/storage/ledger/logs.go @@ -9,10 +9,8 @@ import ( "github.com/formancehq/ledger/pkg/features" "errors" - "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/platform/postgres" "github.com/formancehq/go-libs/v2/pointer" - "github.com/formancehq/go-libs/v2/query" ledger "github.com/formancehq/ledger/internal" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" ) @@ -46,18 +44,18 @@ func (j RawMessage) Value() (driver.Value, error) { return string(j), nil } -func (s *Store) InsertLog(ctx context.Context, log *ledger.Log) error { +func (store *Store) InsertLog(ctx context.Context, log *ledger.Log) error { _, err := tracing.TraceWithMetric( ctx, "InsertLog", - s.tracer, - s.insertLogHistogram, + store.tracer, + store.insertLogHistogram, tracing.NoResult(func(ctx context.Context) error { // We lock logs table as we need than the last log does not change until the transaction commit - if s.ledger.HasFeature(features.FeatureHashLogs, "SYNC") { - _, err := s.db.NewRaw(`select pg_advisory_xact_lock(?)`, s.ledger.ID).Exec(ctx) + if store.ledger.HasFeature(features.FeatureHashLogs, "SYNC") { + _, err := store.db.NewRaw(`select pg_advisory_xact_lock(?)`, store.ledger.ID).Exec(ctx) if err != nil { return postgres.ResolveError(err) } @@ -78,19 +76,19 @@ func (s *Store) InsertLog(ctx context.Context, log *ledger.Log) error { return err } - query := s.db. + query := store.db. NewInsert(). Model(&Log{ Log: log, - Ledger: s.ledger.Name, + Ledger: store.ledger.Name, Data: payloadData, Memento: mementoData, }). - ModelTableExpr(s.GetPrefixedRelationName("logs")). + ModelTableExpr(store.GetPrefixedRelationName("logs")). Returning("*") if log.ID == 0 { - query = query.Value("id", "nextval(?)", s.GetPrefixedRelationName(fmt.Sprintf(`"log_id_%d"`, s.ledger.ID))) + query = query.Value("id", "nextval(?)", store.GetPrefixedRelationName(fmt.Sprintf(`"log_id_%d"`, store.ledger.ID))) } _, err = query.Exec(ctx) @@ -113,57 +111,20 @@ func (s *Store) InsertLog(ctx context.Context, log *ledger.Log) error { return err } -func (s *Store) ListLogs(ctx context.Context, q ledgercontroller.GetLogsQuery) (*bunpaginate.Cursor[ledger.Log], error) { - return tracing.TraceWithMetric( - ctx, - "ListLogs", - s.tracer, - s.listLogsHistogram, - func(ctx context.Context) (*bunpaginate.Cursor[ledger.Log], error) { - selectQuery := s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("logs")). - ColumnExpr("*"). - Where("ledger = ?", s.ledger.Name) - - if q.Options.QueryBuilder != nil { - subQuery, args, err := q.Options.QueryBuilder.Build(query.ContextFn(func(key, operator string, value any) (string, []any, error) { - switch { - case key == "date": - return fmt.Sprintf("%s %s ?", key, query.DefaultComparisonOperatorsMapping[operator]), []any{value}, nil - default: - return "", nil, fmt.Errorf("unknown key '%s' when building query", key) - } - })) - if err != nil { - return nil, err - } - selectQuery = selectQuery.Where(subQuery, args...) - } - - cursor, err := bunpaginate.UsingColumn[ledgercontroller.PaginatedQueryOptions[any], Log](ctx, selectQuery, bunpaginate.ColumnPaginatedQuery[ledgercontroller.PaginatedQueryOptions[any]](q)) - if err != nil { - return nil, err - } - - return bunpaginate.MapCursor(cursor, Log.ToCore), nil - }, - ) -} - -func (s *Store) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*ledger.Log, error) { +func (store *Store) ReadLogWithIdempotencyKey(ctx context.Context, key string) (*ledger.Log, error) { return tracing.TraceWithMetric( ctx, "ReadLogWithIdempotencyKey", - s.tracer, - s.readLogWithIdempotencyKeyHistogram, + store.tracer, + store.readLogWithIdempotencyKeyHistogram, func(ctx context.Context) (*ledger.Log, error) { ret := &Log{} - if err := s.db.NewSelect(). + if err := store.db.NewSelect(). Model(ret). - ModelTableExpr(s.GetPrefixedRelationName("logs")). + ModelTableExpr(store.GetPrefixedRelationName("logs")). Column("*"). Where("idempotency_key = ?", key). - Where("ledger = ?", s.ledger.Name). + Where("ledger = ?", store.ledger.Name). Limit(1). Scan(ctx); err != nil { return nil, postgres.ResolveError(err) diff --git a/internal/storage/ledger/logs_test.go b/internal/storage/ledger/logs_test.go index a4b284a6b..bb9341e72 100644 --- a/internal/storage/ledger/logs_test.go +++ b/internal/storage/ledger/logs_test.go @@ -5,6 +5,7 @@ package ledger_test import ( "context" "database/sql" + "github.com/formancehq/go-libs/v2/pointer" "golang.org/x/sync/errgroup" "math/big" "testing" @@ -120,9 +121,10 @@ func TestInsertLog(t *testing.T) { err := errGroup.Wait() require.NoError(t, err) - logs, err := store.ListLogs(ctx, ledgercontroller.NewListLogsQuery(ledgercontroller.PaginatedQueryOptions[any]{ + logs, err := store.Logs().Paginate(ctx, ledgercontroller.ColumnPaginatedQuery[any]{ PageSize: countLogs, - }).WithOrder(bunpaginate.OrderAsc)) + Order: pointer.For(bunpaginate.Order(bunpaginate.OrderAsc)), + }) require.NoError(t, err) var previous *ledger.Log @@ -180,26 +182,30 @@ func TestLogsList(t *testing.T) { require.NoError(t, err) } - cursor, err := store.ListLogs(context.Background(), ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil))) + cursor, err := store.Logs().Paginate(context.Background(), ledgercontroller.ColumnPaginatedQuery[any]{}) require.NoError(t, err) require.Equal(t, bunpaginate.QueryDefaultPageSize, cursor.PageSize) require.Equal(t, 3, len(cursor.Data)) require.EqualValues(t, 3, cursor.Data[0].ID) - cursor, err = store.ListLogs(context.Background(), ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil).WithPageSize(1))) + cursor, err = store.Logs().Paginate(context.Background(), ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: 1, + }) require.NoError(t, err) // Should get only the first log. require.Equal(t, 1, cursor.PageSize) require.EqualValues(t, 3, cursor.Data[0].ID) - cursor, err = store.ListLogs(context.Background(), ledgercontroller.NewListLogsQuery(ledgercontroller.NewPaginatedQueryOptions[any](nil). - WithQueryBuilder(query.And( - query.Gte("date", now.Add(-2*time.Hour)), - query.Lt("date", now.Add(-time.Hour)), - )). - WithPageSize(10), - )) + cursor, err = store.Logs().Paginate(context.Background(), ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: 10, + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.And( + query.Gte("date", now.Add(-2*time.Hour)), + query.Lt("date", now.Add(-time.Hour)), + ), + }, + }) require.NoError(t, err) require.Equal(t, 10, cursor.PageSize) // Should get only the second log, as StartTime is inclusive and EndTime exclusive. diff --git a/internal/storage/ledger/moves.go b/internal/storage/ledger/moves.go index 6f5e55343..00dbc86cc 100644 --- a/internal/storage/ledger/moves.go +++ b/internal/storage/ledger/moves.go @@ -2,80 +2,22 @@ package ledger import ( "context" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/formancehq/ledger/pkg/features" - "github.com/formancehq/go-libs/v2/platform/postgres" - "github.com/formancehq/go-libs/v2/time" ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/ledger/internal/tracing" - "github.com/uptrace/bun" ) -func (s *Store) SortMovesBySeq(date *time.Time) (*bun.SelectQuery, error) { - - ret := s.db.NewSelect() - if !s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { - return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory) - } - - ret = ret. - ModelTableExpr(s.GetPrefixedRelationName("moves")). - Where("ledger = ?", s.ledger.Name). - Order("seq desc") - - if date != nil && !date.IsZero() { - ret = ret.Where("insertion_date <= ?", date) - } - - return ret, nil -} - -func (s *Store) SelectDistinctMovesBySeq(date *time.Time) (*bun.SelectQuery, error) { - sortMovesBySeq, err := s.SortMovesBySeq(date) - if err != nil { - return nil, err - } - ret := s.db.NewSelect(). - TableExpr("(?) moves", sortMovesBySeq). - DistinctOn("accounts_address, asset"). - Column("accounts_address", "asset"). - ColumnExpr("first_value(post_commit_volumes) over (partition by (accounts_address, asset) order by seq desc) as post_commit_volumes"). - Where("ledger = ?", s.ledger.Name) - - if date != nil && !date.IsZero() { - ret = ret.Where("insertion_date <= ?", date) - } - - return ret, nil -} - -func (s *Store) SelectDistinctMovesByEffectiveDate(date *time.Time) *bun.SelectQuery { - ret := s.db.NewSelect(). - TableExpr(s.GetPrefixedRelationName("moves")). - DistinctOn("accounts_address, asset"). - Column("accounts_address", "asset"). - ColumnExpr("first_value(post_commit_effective_volumes) over (partition by (accounts_address, asset) order by effective_date desc, seq desc) as post_commit_effective_volumes"). - Where("ledger = ?", s.ledger.Name) - - if date != nil && !date.IsZero() { - ret = ret.Where("effective_date <= ?", date) - } - - return ret -} - -func (s *Store) InsertMoves(ctx context.Context, moves ...*ledger.Move) error { +func (store *Store) InsertMoves(ctx context.Context, moves ...*ledger.Move) error { _, err := tracing.TraceWithMetric( ctx, "InsertMoves", - s.tracer, - s.insertMovesHistogram, + store.tracer, + store.insertMovesHistogram, tracing.NoResult(func(ctx context.Context) error { - _, err := s.db.NewInsert(). + _, err := store.db.NewInsert(). Model(&moves). - Value("ledger", "?", s.ledger.Name). - ModelTableExpr(s.GetPrefixedRelationName("moves")). + Value("ledger", "?", store.ledger.Name). + ModelTableExpr(store.GetPrefixedRelationName("moves")). Returning("post_commit_volumes, post_commit_effective_volumes"). Exec(ctx) diff --git a/internal/storage/ledger/moves_test.go b/internal/storage/ledger/moves_test.go index 02ace80c4..4d4bb90c4 100644 --- a/internal/storage/ledger/moves_test.go +++ b/internal/storage/ledger/moves_test.go @@ -5,7 +5,6 @@ package ledger_test import ( "database/sql" "fmt" - "github.com/formancehq/go-libs/v2/pointer" "math/big" "math/rand" "testing" @@ -171,14 +170,19 @@ func TestMovesInsert(t *testing.T) { } wp.StopAndWait() - aggregatedBalances, err := store.GetAggregatedBalances(ctx, ledgercontroller.NewGetAggregatedBalancesQuery(ledgercontroller.PITFilter{ - // By using a PIT, we force the usage of the moves table. - // If it was not specified, the test would not been correct. - PIT: pointer.For(time.Now()), - }, nil, true)) + aggregatedVolumes, err := store.AggregatedVolumes().GetOne(ctx, ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions]{ + Opts: ledgercontroller.GetAggregatedVolumesOptions{ + UseInsertionDate: true, + }, + }) require.NoError(t, err) - RequireEqual(t, ledger.BalancesByAssets{ - "USD": big.NewInt(0), - }, aggregatedBalances) + RequireEqual(t, ledger.AggregatedVolumes{ + Aggregated: ledger.VolumesByAssets{ + "USD": { + Input: big.NewInt(1000), + Output: big.NewInt(1000), + }, + }, + }, *aggregatedVolumes) }) } diff --git a/internal/storage/ledger/paginator.go b/internal/storage/ledger/paginator.go new file mode 100644 index 000000000..b73b9f0f6 --- /dev/null +++ b/internal/storage/ledger/paginator.go @@ -0,0 +1,11 @@ +package ledger + +import ( + "github.com/formancehq/go-libs/v2/bun/bunpaginate" + "github.com/uptrace/bun" +) + +type paginator[ResourceType any, PaginationOptions any] interface { + paginate(selectQuery *bun.SelectQuery, opts PaginationOptions) (*bun.SelectQuery, error) + buildCursor(ret []ResourceType, opts PaginationOptions) (*bunpaginate.Cursor[ResourceType], error) +} diff --git a/internal/storage/ledger/paginator_column.go b/internal/storage/ledger/paginator_column.go new file mode 100644 index 000000000..e9ab8c35b --- /dev/null +++ b/internal/storage/ledger/paginator_column.go @@ -0,0 +1,240 @@ +package ledger + +import ( + "fmt" + "github.com/formancehq/go-libs/v2/bun/bunpaginate" + "github.com/formancehq/go-libs/v2/time" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/uptrace/bun" + "math/big" + "reflect" + "strings" + libtime "time" +) + +type columnPaginator[ResourceType, OptionsType any] struct { + defaultPaginationColumn string + defaultOrder bunpaginate.Order +} + +//nolint:unused +func (o columnPaginator[ResourceType, OptionsType]) paginate(sb *bun.SelectQuery, query ledgercontroller.ColumnPaginatedQuery[OptionsType]) (*bun.SelectQuery, error) { + + paginationColumn := o.defaultPaginationColumn + originalOrder := o.defaultOrder + if query.Order != nil { + originalOrder = *query.Order + } + + pageSize := query.PageSize + if pageSize == 0 { + pageSize = bunpaginate.QueryDefaultPageSize + } + + sb = sb.Limit(int(pageSize) + 1) // Fetch one additional item to find the next token + order := originalOrder + if query.Reverse { + order = originalOrder.Reverse() + } + orderExpression := fmt.Sprintf("%s %s", paginationColumn, order) + sb = sb.ColumnExpr("row_number() OVER (ORDER BY " + orderExpression + ")") + + if query.PaginationID != nil { + if query.Reverse { + switch originalOrder { + case bunpaginate.OrderAsc: + sb = sb.Where(fmt.Sprintf("%s < ?", paginationColumn), query.PaginationID) + case bunpaginate.OrderDesc: + sb = sb.Where(fmt.Sprintf("%s > ?", paginationColumn), query.PaginationID) + } + } else { + switch originalOrder { + case bunpaginate.OrderAsc: + sb = sb.Where(fmt.Sprintf("%s >= ?", paginationColumn), query.PaginationID) + case bunpaginate.OrderDesc: + sb = sb.Where(fmt.Sprintf("%s <= ?", paginationColumn), query.PaginationID) + } + } + } + + return sb, nil +} + +//nolint:unused +func (o columnPaginator[ResourceType, OptionsType]) buildCursor(ret []ResourceType, query ledgercontroller.ColumnPaginatedQuery[OptionsType]) (*bunpaginate.Cursor[ResourceType], error) { + + paginationColumn := query.Column + if paginationColumn == "" { + paginationColumn = o.defaultPaginationColumn + } + + pageSize := query.PageSize + if pageSize == 0 { + pageSize = bunpaginate.QueryDefaultPageSize + } + + order := o.defaultOrder + if query.Order != nil { + order = *query.Order + } + + var v ResourceType + fields := findPaginationFieldPath(v, paginationColumn) + + var ( + paginationIDs = make([]*big.Int, 0) + ) + for _, t := range ret { + paginationID := findPaginationField(t, fields...) + if query.Bottom == nil { + query.Bottom = paginationID + } + paginationIDs = append(paginationIDs, paginationID) + } + + hasMore := len(ret) > int(pageSize) + if hasMore { + ret = ret[:len(ret)-1] + } + if query.Reverse { + for i := 0; i < len(ret)/2; i++ { + ret[i], ret[len(ret)-i-1] = ret[len(ret)-i-1], ret[i] + } + } + + var previous, next *ledgercontroller.ColumnPaginatedQuery[OptionsType] + + if query.Reverse { + cp := query + cp.Reverse = false + next = &cp + + if hasMore { + cp := query + cp.PaginationID = paginationIDs[len(paginationIDs)-2] + previous = &cp + } + } else { + if hasMore { + cp := query + cp.PaginationID = paginationIDs[len(paginationIDs)-1] + next = &cp + } + if query.PaginationID != nil { + if (order == bunpaginate.OrderAsc && query.PaginationID.Cmp(query.Bottom) > 0) || (order == bunpaginate.OrderDesc && query.PaginationID.Cmp(query.Bottom) < 0) { + cp := query + cp.Reverse = true + previous = &cp + } + } + } + + return &bunpaginate.Cursor[ResourceType]{ + PageSize: int(pageSize), + HasMore: next != nil, + Previous: encodeCursor[OptionsType, ledgercontroller.ColumnPaginatedQuery[OptionsType]](previous), + Next: encodeCursor[OptionsType, ledgercontroller.ColumnPaginatedQuery[OptionsType]](next), + Data: ret, + }, nil +} + +var _ paginator[any, ledgercontroller.ColumnPaginatedQuery[any]] = &columnPaginator[any, any]{} + +//nolint:unused +func findPaginationFieldPath(v any, paginationColumn string) []reflect.StructField { + + typeOfT := reflect.TypeOf(v) + for i := 0; i < typeOfT.NumField(); i++ { + field := typeOfT.Field(i) + fieldType := field.Type + + // If the field is a pointer, we unreference it to target the concrete type + // For example: + // type Object struct { + // *AnotherObject + // } + for { + if field.Type.Kind() == reflect.Ptr { + fieldType = field.Type.Elem() + } + break + } + + switch fieldType.Kind() { + case reflect.Struct: + if fieldType.AssignableTo(reflect.TypeOf(time.Time{})) || + fieldType.AssignableTo(reflect.TypeOf(libtime.Time{})) || + fieldType.AssignableTo(reflect.TypeOf(big.Int{})) || + fieldType.AssignableTo(reflect.TypeOf(bunpaginate.BigInt{})) { + + if fields := checkTag(field, paginationColumn); len(fields) > 0 { + return fields + } + } else { + fields := findPaginationFieldPath(reflect.New(fieldType).Elem().Interface(), paginationColumn) + if len(fields) > 0 { + return fields + } + } + default: + if fields := checkTag(field, paginationColumn); len(fields) > 0 { + return fields + } + } + } + + return nil +} + +//nolint:unused +func checkTag(field reflect.StructField, paginationColumn string) []reflect.StructField { + tag := field.Tag.Get("bun") + column := strings.Split(tag, ",")[0] + if column == paginationColumn { + return []reflect.StructField{field} + } + + return nil +} + +//nolint:unused +func findPaginationField(v any, fields ...reflect.StructField) *big.Int { + vOf := reflect.ValueOf(v) + field := vOf.FieldByName(fields[0].Name) + if len(fields) == 1 { + switch rawPaginationID := field.Interface().(type) { + case time.Time: + return big.NewInt(rawPaginationID.UTC().UnixMicro()) + case *time.Time: + return big.NewInt(rawPaginationID.UTC().UnixMicro()) + case *libtime.Time: + return big.NewInt(rawPaginationID.UTC().UnixMicro()) + case libtime.Time: + return big.NewInt(rawPaginationID.UTC().UnixMicro()) + case *bunpaginate.BigInt: + return (*big.Int)(rawPaginationID) + case bunpaginate.BigInt: + return (*big.Int)(&rawPaginationID) + case *big.Int: + return rawPaginationID + case big.Int: + return &rawPaginationID + case int64: + return big.NewInt(rawPaginationID) + case int: + return big.NewInt(int64(rawPaginationID)) + default: + panic(fmt.Sprintf("invalid paginationID, type %T not handled", rawPaginationID)) + } + } + + return findPaginationField(v, fields[1:]...) +} + +//nolint:unused +func encodeCursor[OptionsType any, PaginatedQueryType ledgercontroller.PaginatedQuery[OptionsType]](v *PaginatedQueryType) string { + if v == nil { + return "" + } + return bunpaginate.EncodeCursor(v) +} diff --git a/internal/storage/ledger/paginator_offset.go b/internal/storage/ledger/paginator_offset.go new file mode 100644 index 000000000..7fcc09537 --- /dev/null +++ b/internal/storage/ledger/paginator_offset.go @@ -0,0 +1,79 @@ +package ledger + +import ( + "fmt" + "github.com/formancehq/go-libs/v2/bun/bunpaginate" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/uptrace/bun" + "math" +) + +type offsetPaginator[ResourceType, OptionsType any] struct { + defaultPaginationColumn string + defaultOrder bunpaginate.Order +} + +//nolint:unused +func (o offsetPaginator[ResourceType, OptionsType]) paginate(sb *bun.SelectQuery, query ledgercontroller.OffsetPaginatedQuery[OptionsType]) (*bun.SelectQuery, error) { + + paginationColumn := o.defaultPaginationColumn + originalOrder := o.defaultOrder + if query.Order != nil { + originalOrder = *query.Order + } + + orderExpression := fmt.Sprintf("%s %s", paginationColumn, originalOrder) + sb = sb.ColumnExpr("row_number() OVER (ORDER BY " + orderExpression + ")") + + if query.Offset > math.MaxInt32 { + return nil, fmt.Errorf("offset value exceeds maximum allowed value") + } + if query.Offset > 0 { + sb = sb.Offset(int(query.Offset)) + } + + if query.PageSize > 0 { + sb = sb.Limit(int(query.PageSize) + 1) + } + + return sb, nil +} + +//nolint:unused +func (o offsetPaginator[ResourceType, OptionsType]) buildCursor(ret []ResourceType, query ledgercontroller.OffsetPaginatedQuery[OptionsType]) (*bunpaginate.Cursor[ResourceType], error) { + + var previous, next *ledgercontroller.OffsetPaginatedQuery[OptionsType] + + // Page with transactions before + if query.Offset > 0 { + cp := query + offset := int(query.Offset) - int(query.PageSize) + if offset < 0 { + offset = 0 + } + cp.Offset = uint64(offset) + previous = &cp + } + + // Page with transactions after + if query.PageSize != 0 && len(ret) > int(query.PageSize) { + cp := query + // Check for potential overflow + if query.Offset > math.MaxUint64-query.PageSize { + return nil, fmt.Errorf("offset overflow") + } + cp.Offset = query.Offset + query.PageSize + next = &cp + ret = ret[:len(ret)-1] + } + + return &bunpaginate.Cursor[ResourceType]{ + PageSize: int(query.PageSize), + HasMore: next != nil, + Previous: encodeCursor[OptionsType, ledgercontroller.OffsetPaginatedQuery[OptionsType]](previous), + Next: encodeCursor[OptionsType, ledgercontroller.OffsetPaginatedQuery[OptionsType]](next), + Data: ret, + }, nil +} + +var _ paginator[any, ledgercontroller.OffsetPaginatedQuery[any]] = &offsetPaginator[any, any]{} diff --git a/internal/storage/ledger/resource.go b/internal/storage/ledger/resource.go new file mode 100644 index 000000000..bb074a120 --- /dev/null +++ b/internal/storage/ledger/resource.go @@ -0,0 +1,364 @@ +package ledger + +import ( + "context" + "fmt" + "github.com/formancehq/go-libs/v2/bun/bunpaginate" + "github.com/formancehq/go-libs/v2/platform/postgres" + "github.com/formancehq/go-libs/v2/pointer" + "github.com/formancehq/go-libs/v2/query" + ledger "github.com/formancehq/ledger/internal" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/uptrace/bun" + "regexp" + "slices" +) + +func convertOperatorToSQL(operator string) string { + switch operator { + case "$match": + return "=" + case "$lt": + return "<" + case "$gt": + return ">" + case "$lte": + return "<=" + case "$gte": + return ">=" + } + panic("unreachable") +} + +type joinCondition struct { + left string + right string +} + +type propertyValidator interface { + validate(ledger.Ledger, string, string, any) error +} +type propertyValidatorFunc func(ledger.Ledger, string, string, any) error + +func (p propertyValidatorFunc) validate(l ledger.Ledger, operator string, key string, value any) error { + return p(l, operator, key, value) +} + +func acceptOperators(operators ...string) propertyValidator { + return propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + if !slices.Contains(operators, operator) { + return ledgercontroller.NewErrInvalidQuery("operator '%s' is not allowed", operator) + } + return nil + }) +} + +type filter struct { + name string + aliases []string + matchers []func(key string) bool + validators []propertyValidator +} + +type repositoryHandlerBuildContext[Opts any] struct { + ledgercontroller.ResourceQuery[Opts] + filters map[string]any +} + +func (ctx repositoryHandlerBuildContext[Opts]) useFilter(v string, matchers ...func(value any) bool) bool { + value, ok := ctx.filters[v] + if !ok { + return false + } + for _, matcher := range matchers { + if !matcher(value) { + return false + } + } + + return true +} + +type repositoryHandler[Opts any] interface { + filters() []filter + buildDataset(store *Store, query repositoryHandlerBuildContext[Opts]) (*bun.SelectQuery, error) + resolveFilter(store *Store, query ledgercontroller.ResourceQuery[Opts], operator, property string, value any) (string, []any, error) + project(store *Store, query ledgercontroller.ResourceQuery[Opts], selectQuery *bun.SelectQuery) (*bun.SelectQuery, error) + expand(store *Store, query ledgercontroller.ResourceQuery[Opts], property string) (*bun.SelectQuery, *joinCondition, error) +} + +type resourceRepository[ResourceType, OptionsType any] struct { + resourceHandler repositoryHandler[OptionsType] + store *Store + ledger ledger.Ledger +} + +func (r *resourceRepository[ResourceType, OptionsType]) validateFilters(builder query.Builder) (map[string]any, error) { + if builder == nil { + return nil, nil + } + + ret := make(map[string]any) + properties := r.resourceHandler.filters() + if err := builder.Walk(func(operator string, key string, value any) (err error) { + + found := false + for _, property := range properties { + if len(property.matchers) > 0 { + for _, matcher := range property.matchers { + if found = matcher(key); found { + break + } + } + } else { + options := append([]string{property.name}, property.aliases...) + for _, option := range options { + if found, err = regexp.MatchString("^"+option+"$", key); err != nil { + return fmt.Errorf("failed to match regex for key '%s': %w", key, err) + } else if found { + break + } + } + } + if !found { + continue + } + + for _, validator := range property.validators { + if err := validator.validate(r.ledger, operator, key, value); err != nil { + return err + } + } + ret[property.name] = value + break + } + + if !found { + return ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) + } + + return nil + }); err != nil { + return nil, err + } + + return ret, nil +} + +func (r *resourceRepository[ResourceType, OptionsType]) buildFilteredDataset(q ledgercontroller.ResourceQuery[OptionsType]) (*bun.SelectQuery, error) { + + filters, err := r.validateFilters(q.Builder) + if err != nil { + return nil, err + } + + dataset, err := r.resourceHandler.buildDataset(r.store, repositoryHandlerBuildContext[OptionsType]{ + ResourceQuery: q, + filters: filters, + }) + if err != nil { + return nil, err + } + + dataset = r.store.db.NewSelect(). + ModelTableExpr("(?) dataset", dataset) + + if q.Builder != nil { + // Convert filters to where clause + where, args, err := q.Builder.Build(query.ContextFn(func(key, operator string, value any) (string, []any, error) { + return r.resourceHandler.resolveFilter(r.store, q, operator, key, value) + })) + if err != nil { + return nil, err + } + if len(args) > 0 { + dataset = dataset.Where(where, args...) + } else { + dataset = dataset.Where(where) + } + } + + return r.resourceHandler.project(r.store, q, dataset) +} + +func (r *resourceRepository[ResourceType, OptionsType]) expand(dataset *bun.SelectQuery, q ledgercontroller.ResourceQuery[OptionsType]) (*bun.SelectQuery, error) { + dataset = r.store.db.NewSelect(). + With("dataset", dataset). + ModelTableExpr("dataset"). + ColumnExpr("*") + + slices.Sort(q.Expand) + + for i, expand := range q.Expand { + selectQuery, joinCondition, err := r.resourceHandler.expand(r.store, q, expand) + if err != nil { + return nil, err + } + + if selectQuery == nil { + continue + } + + expandCTEName := fmt.Sprintf("expand%d", i) + dataset = dataset. + With(expandCTEName, selectQuery). + Join(fmt.Sprintf( + "left join %s on %s.%s = dataset.%s", + expandCTEName, + expandCTEName, + joinCondition.right, + joinCondition.left, + )) + } + + return dataset, nil +} + +func (r *resourceRepository[ResourceType, OptionsType]) GetOne(ctx context.Context, query ledgercontroller.ResourceQuery[OptionsType]) (*ResourceType, error) { + + finalQuery, err := r.buildFilteredDataset(query) + if err != nil { + return nil, err + } + + finalQuery, err = r.expand(finalQuery, query) + if err != nil { + return nil, err + } + + ret := make([]ResourceType, 0) + if err := finalQuery. + Model(&ret). + Limit(1). + Scan(ctx); err != nil { + return nil, err + } + if len(ret) == 0 { + return nil, postgres.ErrNotFound + } + + return &ret[0], nil +} + +func (r *resourceRepository[ResourceType, OptionsType]) Count(ctx context.Context, query ledgercontroller.ResourceQuery[OptionsType]) (int, error) { + + finalQuery, err := r.buildFilteredDataset(query) + if err != nil { + return 0, err + } + + count, err := finalQuery.Count(ctx) + return count, postgres.ResolveError(err) +} + +func newResourceRepository[ResourceType, OptionsType any]( + store *Store, + l ledger.Ledger, + handler repositoryHandler[OptionsType], +) *resourceRepository[ResourceType, OptionsType] { + return &resourceRepository[ResourceType, OptionsType]{ + resourceHandler: handler, + store: store, + ledger: l, + } +} + +type paginatedResourceRepository[ResourceType, OptionsType any, PaginationQueryType ledgercontroller.PaginatedQuery[OptionsType]] struct { + *resourceRepository[ResourceType, OptionsType] + paginator paginator[ResourceType, PaginationQueryType] +} + +func (r *paginatedResourceRepository[ResourceType, OptionsType, PaginationQueryType]) Paginate( + ctx context.Context, + paginationOptions PaginationQueryType, +) (*bunpaginate.Cursor[ResourceType], error) { + + var resourceQuery ledgercontroller.ResourceQuery[OptionsType] + switch v := any(paginationOptions).(type) { + case ledgercontroller.OffsetPaginatedQuery[OptionsType]: + resourceQuery = v.Options + case ledgercontroller.ColumnPaginatedQuery[OptionsType]: + resourceQuery = v.Options + default: + panic("should not happen") + } + + finalQuery, err := r.buildFilteredDataset(resourceQuery) + if err != nil { + return nil, fmt.Errorf("building filtered dataset: %w", err) + } + + finalQuery, err = r.paginator.paginate(finalQuery, paginationOptions) + if err != nil { + return nil, fmt.Errorf("paginating request: %w", err) + } + + finalQuery, err = r.expand(finalQuery, resourceQuery) + if err != nil { + return nil, fmt.Errorf("expanding results: %w", err) + } + finalQuery = finalQuery.Order("row_number") + + ret := make([]ResourceType, 0) + err = finalQuery.Model(&ret).Scan(ctx) + if err != nil { + return nil, fmt.Errorf("scanning results: %w", err) + } + + return r.paginator.buildCursor(ret, paginationOptions) +} + +func newPaginatedResourceRepository[ResourceType, OptionsType any, PaginationQueryType ledgercontroller.PaginatedQuery[OptionsType]]( + store *Store, + l ledger.Ledger, + handler repositoryHandler[OptionsType], + paginator paginator[ResourceType, PaginationQueryType], +) *paginatedResourceRepository[ResourceType, OptionsType, PaginationQueryType] { + return &paginatedResourceRepository[ResourceType, OptionsType, PaginationQueryType]{ + resourceRepository: newResourceRepository[ResourceType, OptionsType](store, l, handler), + paginator: paginator, + } +} + +type paginatedResourceRepositoryMapper[ToResourceType any, OriginalResourceType interface { + ToCore() ToResourceType +}, OptionsType any, PaginationQueryType ledgercontroller.PaginatedQuery[OptionsType]] struct { + *paginatedResourceRepository[OriginalResourceType, OptionsType, PaginationQueryType] +} + +func (m paginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType]) Paginate( + ctx context.Context, + paginationOptions PaginationQueryType, +) (*bunpaginate.Cursor[ToResourceType], error) { + cursor, err := m.paginatedResourceRepository.Paginate(ctx, paginationOptions) + if err != nil { + return nil, err + } + + return bunpaginate.MapCursor(cursor, OriginalResourceType.ToCore), nil +} + +func (m paginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType]) GetOne( + ctx context.Context, + query ledgercontroller.ResourceQuery[OptionsType], +) (*ToResourceType, error) { + item, err := m.paginatedResourceRepository.GetOne(ctx, query) + if err != nil { + return nil, err + } + + return pointer.For((*item).ToCore()), nil +} + +func newPaginatedResourceRepositoryMapper[ToResourceType any, OriginalResourceType interface { + ToCore() ToResourceType +}, OptionsType any, PaginationQueryType ledgercontroller.PaginatedQuery[OptionsType]]( + store *Store, + l ledger.Ledger, + handler repositoryHandler[OptionsType], + paginator paginator[OriginalResourceType, PaginationQueryType], +) *paginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType] { + return &paginatedResourceRepositoryMapper[ToResourceType, OriginalResourceType, OptionsType, PaginationQueryType]{ + paginatedResourceRepository: newPaginatedResourceRepository(store, l, handler, paginator), + } +} diff --git a/internal/storage/ledger/resource_accounts.go b/internal/storage/ledger/resource_accounts.go new file mode 100644 index 000000000..6a0a7f5d9 --- /dev/null +++ b/internal/storage/ledger/resource_accounts.go @@ -0,0 +1,187 @@ +package ledger + +import ( + "fmt" + ledger "github.com/formancehq/ledger/internal" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/formancehq/ledger/pkg/features" + "github.com/stoewer/go-strcase" + "github.com/uptrace/bun" +) + +type accountsResourceHandler struct{} + +func (h accountsResourceHandler) filters() []filter { + return []filter{ + { + name: "address", + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + return validateAddressFilter(l, operator, value) + }), + }, + }, + { + name: "first_usage", + validators: []propertyValidator{ + acceptOperators("$lt", "$gt", "$lte", "$gte", "$match"), + }, + }, + { + name: `balance(\[.*])?`, + validators: []propertyValidator{ + acceptOperators("$lt", "$gt", "$lte", "$gte", "$match"), + }, + }, + { + name: "metadata", + validators: []propertyValidator{ + acceptOperators("$exists"), + }, + }, + { + name: `metadata\[.*]`, + validators: []propertyValidator{ + acceptOperators("$match"), + }, + }, + } +} + +func (h accountsResourceHandler) buildDataset(store *Store, opts repositoryHandlerBuildContext[any]) (*bun.SelectQuery, error) { + ret := store.db.NewSelect() + + // Build the query + ret = ret. + ModelTableExpr(store.GetPrefixedRelationName("accounts")). + Column("address", "address_array", "first_usage", "insertion_date", "updated_at"). + Where("ledger = ?", store.ledger.Name) + + if opts.PIT != nil && !opts.PIT.IsZero() { + ret = ret.Where("accounts.first_usage <= ?", opts.PIT) + } + + if store.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && opts.PIT != nil && !opts.PIT.IsZero() { + selectDistinctAccountMetadataHistories := store.db.NewSelect(). + DistinctOn("accounts_address"). + ModelTableExpr(store.GetPrefixedRelationName("accounts_metadata")). + Where("ledger = ?", store.ledger.Name). + Column("accounts_address"). + ColumnExpr("first_value(metadata) over (partition by accounts_address order by revision desc) as metadata"). + Where("date <= ?", opts.PIT) + + ret = ret. + Join( + `left join (?) accounts_metadata on accounts_metadata.accounts_address = accounts.address`, + selectDistinctAccountMetadataHistories, + ). + ColumnExpr("coalesce(accounts_metadata.metadata, '{}'::jsonb) as metadata") + } else { + ret = ret.ColumnExpr("accounts.metadata") + } + + return ret, nil +} + +func (h accountsResourceHandler) resolveFilter(store *Store, opts ledgercontroller.ResourceQuery[any], operator, property string, value any) (string, []any, error) { + switch { + case property == "address": + return filterAccountAddress(value.(string), "address"), nil, nil + case property == "first_usage": + return fmt.Sprintf("first_usage %s ?", convertOperatorToSQL(operator)), []any{value}, nil + case balanceRegex.MatchString(property) || property == "balance": + + selectBalance := store.db.NewSelect(). + Where("accounts_address = dataset.address"). + Where("ledger = ?", store.ledger.Name) + + if opts.PIT != nil && !opts.PIT.IsZero() { + if !store.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + return "", nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory) + } + selectBalance = selectBalance. + ModelTableExpr(store.GetPrefixedRelationName("moves")). + DistinctOn("asset"). + ColumnExpr("first_value((post_commit_volumes).inputs - (post_commit_volumes).outputs) over (partition by (accounts_address, asset) order by seq desc) as balance"). + Where("insertion_date <= ?", opts.PIT) + } else { + selectBalance = selectBalance. + ModelTableExpr(store.GetPrefixedRelationName("accounts_volumes")). + ColumnExpr("input - output as balance") + } + + if balanceRegex.MatchString(property) { + selectBalance = selectBalance.Where("asset = ?", balanceRegex.FindAllStringSubmatch(property, 2)[0][1]) + } + + return store.db.NewSelect(). + TableExpr("(?) balance", selectBalance). + ColumnExpr(fmt.Sprintf("balance %s ?", convertOperatorToSQL(operator)), value). + String(), nil, nil + case property == "metadata": + return "metadata -> ? is not null", []any{value}, nil + + case metadataRegex.Match([]byte(property)): + match := metadataRegex.FindAllStringSubmatch(property, 3) + + return "metadata @> ?", []any{map[string]any{ + match[0][1]: value, + }}, nil + default: + return "", nil, ledgercontroller.NewErrInvalidQuery("invalid filter property %s", property) + } +} + +func (h accountsResourceHandler) project(store *Store, query ledgercontroller.ResourceQuery[any], selectQuery *bun.SelectQuery) (*bun.SelectQuery, error) { + return selectQuery.ColumnExpr("*"), nil +} + +func (h accountsResourceHandler) expand(store *Store, opts ledgercontroller.ResourceQuery[any], property string) (*bun.SelectQuery, *joinCondition, error) { + switch property { + case "volumes": + if !store.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + return nil, nil, ledgercontroller.NewErrInvalidQuery("feature %s must be 'ON' to use volumes", features.FeatureMovesHistory) + } + case "effectiveVolumes": + if !store.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { + return nil, nil, ledgercontroller.NewErrInvalidQuery("feature %s must be 'SYNC' to use effectiveVolumes", features.FeatureMovesHistoryPostCommitEffectiveVolumes) + } + } + + selectRowsQuery := store.db.NewSelect(). + Where("accounts_address in (select address from dataset)") + if opts.UsePIT() { + selectRowsQuery = selectRowsQuery. + ModelTableExpr(store.GetPrefixedRelationName("moves")). + DistinctOn("accounts_address, asset"). + Column("accounts_address", "asset"). + Where("ledger = ?", store.ledger.Name) + if property == "volumes" { + selectRowsQuery = selectRowsQuery. + ColumnExpr("first_value(post_commit_volumes) over (partition by (accounts_address, asset) order by seq desc) as volumes"). + Where("insertion_date <= ?", opts.PIT) + } else { + selectRowsQuery = selectRowsQuery. + ColumnExpr("first_value(post_commit_volumes) over (partition by (accounts_address, asset) order by effective_date desc) as volumes"). + Where("effective_date <= ?", opts.PIT) + } + } else { + selectRowsQuery = selectRowsQuery. + ModelTableExpr(store.GetPrefixedRelationName("accounts_volumes")). + Column("asset", "accounts_address"). + ColumnExpr("(input, output)::"+store.GetPrefixedRelationName("volumes")+" as volumes"). + Where("ledger = ?", store.ledger.Name) + } + + return store.db.NewSelect(). + With("rows", selectRowsQuery). + ModelTableExpr("rows"). + Column("accounts_address"). + ColumnExpr("public.aggregate_objects(json_build_object(asset, json_build_object('input', (volumes).inputs, 'output', (volumes).outputs))::jsonb) as " + strcase.SnakeCase(property)). + Group("accounts_address"), &joinCondition{ + left: "address", + right: "accounts_address", + }, nil +} + +var _ repositoryHandler[any] = accountsResourceHandler{} diff --git a/internal/storage/ledger/resource_aggregated_balances.go b/internal/storage/ledger/resource_aggregated_balances.go new file mode 100644 index 000000000..b5a32370c --- /dev/null +++ b/internal/storage/ledger/resource_aggregated_balances.go @@ -0,0 +1,172 @@ +package ledger + +import ( + "errors" + "fmt" + ledger "github.com/formancehq/ledger/internal" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/formancehq/ledger/pkg/features" + "github.com/uptrace/bun" +) + +type aggregatedBalancesResourceRepositoryHandler struct{} + +func (h aggregatedBalancesResourceRepositoryHandler) filters() []filter { + return []filter{ + { + name: "address", + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + return validateAddressFilter(l, operator, value) + }), + }, + }, + { + name: "metadata", + matchers: []func(string) bool{ + func(key string) bool { + return key == "metadata" || metadataRegex.Match([]byte(key)) + }, + }, + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + if key == "metadata" { + if operator != "$exists" { + return fmt.Errorf("unsupported operator %s for metadata", operator) + } + return nil + } + if operator != "$match" { + return fmt.Errorf("unsupported operator %s for metadata", operator) + } + return nil + }), + }, + }, + } +} + +func (h aggregatedBalancesResourceRepositoryHandler) buildDataset(store *Store, query repositoryHandlerBuildContext[ledgercontroller.GetAggregatedVolumesOptions]) (*bun.SelectQuery, error) { + + if query.UsePIT() { + ret := store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName("moves")). + DistinctOn("accounts_address, asset"). + Column("accounts_address", "asset"). + Where("ledger = ?", store.ledger.Name) + if query.Opts.UseInsertionDate { + if !store.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory) + } + + ret = ret. + ColumnExpr("first_value(post_commit_volumes) over (partition by (accounts_address, asset) order by seq desc) as volumes"). + Where("insertion_date <= ?", query.PIT) + } else { + if !store.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { + return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes) + } + + ret = ret. + ColumnExpr("first_value(post_commit_effective_volumes) over (partition by (accounts_address, asset) order by effective_date desc, seq desc) as volumes"). + Where("effective_date <= ?", query.PIT) + } + + if query.useFilter("address", isPartialAddress) { + subQuery := store.db.NewSelect(). + TableExpr(store.GetPrefixedRelationName("accounts")). + Column("address_array"). + Where("accounts.address = accounts_address"). + Where("ledger = ?", store.ledger.Name) + + ret = ret. + ColumnExpr("accounts.address_array as accounts_address_array"). + Join(`join lateral (?) accounts on true`, subQuery) + } + + if query.useFilter("metadata") { + subQuery := store.db.NewSelect(). + DistinctOn("accounts_address"). + ModelTableExpr(store.GetPrefixedRelationName("accounts_metadata")). + ColumnExpr("first_value(metadata) over (partition by accounts_address order by revision desc) as metadata"). + Where("ledger = ?", store.ledger.Name). + Where("accounts_metadata.accounts_address = moves.accounts_address"). + Where("date <= ?", query.PIT) + + ret = ret. + Join(`left join lateral (?) accounts_metadata on true`, subQuery). + Column("metadata") + } + + return ret, nil + } else { + ret := store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName("accounts_volumes")). + Column("asset", "accounts_address"). + ColumnExpr("(input, output)::"+store.GetPrefixedRelationName("volumes")+" as volumes"). + Where("ledger = ?", store.ledger.Name) + + if query.useFilter("metadata") || query.useFilter("address", isPartialAddress) { + subQuery := store.db.NewSelect(). + TableExpr(store.GetPrefixedRelationName("accounts")). + Column("address"). + Where("ledger = ?", store.ledger.Name). + Where("accounts.address = accounts_address") + + if query.useFilter("address") { + subQuery = subQuery.ColumnExpr("address_array as accounts_address_array") + ret = ret.Column("accounts_address_array") + } + if query.useFilter("metadata") { + subQuery = subQuery.ColumnExpr("metadata") + ret = ret.Column("metadata") + } + + ret = ret. + Join(`join lateral (?) accounts on true`, subQuery) + } + + return ret, nil + } +} + +func (h aggregatedBalancesResourceRepositoryHandler) resolveFilter(store *Store, query ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions], operator, property string, value any) (string, []any, error) { + switch { + case property == "address": + return filterAccountAddress(value.(string), "accounts_address"), nil, nil + case metadataRegex.Match([]byte(property)) || property == "metadata": + if property == "metadata" { + return "metadata -> ? is not null", []any{value}, nil + } else { + match := metadataRegex.FindAllStringSubmatch(property, 3) + + return "metadata @> ?", []any{map[string]any{ + match[0][1]: value, + }}, nil + } + default: + return "", nil, ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", property) + } +} + +func (h aggregatedBalancesResourceRepositoryHandler) expand(_ *Store, _ ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions], property string) (*bun.SelectQuery, *joinCondition, error) { + return nil, nil, errors.New("no expand available for aggregated balances") +} + +func (h aggregatedBalancesResourceRepositoryHandler) project( + store *Store, + _ ledgercontroller.ResourceQuery[ledgercontroller.GetAggregatedVolumesOptions], + selectQuery *bun.SelectQuery, +) (*bun.SelectQuery, error) { + sumVolumesForAsset := store.db.NewSelect(). + TableExpr("(?) values", selectQuery). + Group("asset"). + Column("asset"). + ColumnExpr("json_build_object('input', sum(((volumes).inputs)::numeric), 'output', sum(((volumes).outputs)::numeric)) as volumes") + + return store.db.NewSelect(). + TableExpr("(?) values", sumVolumesForAsset). + ColumnExpr("aggregate_objects(json_build_object(asset, volumes)::jsonb) as aggregated"), nil +} + +var _ repositoryHandler[ledgercontroller.GetAggregatedVolumesOptions] = aggregatedBalancesResourceRepositoryHandler{} diff --git a/internal/storage/ledger/resource_logs.go b/internal/storage/ledger/resource_logs.go new file mode 100644 index 000000000..3c2b6d361 --- /dev/null +++ b/internal/storage/ledger/resource_logs.go @@ -0,0 +1,45 @@ +package ledger + +import ( + "errors" + "fmt" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/uptrace/bun" +) + +type logsResourceHandler struct{} + +func (h logsResourceHandler) filters() []filter { + return []filter{ + { + // todo: add validators + name: "date", + }, + } +} + +func (h logsResourceHandler) buildDataset(store *Store, _ repositoryHandlerBuildContext[any]) (*bun.SelectQuery, error) { + return store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName("logs")). + ColumnExpr("*"). + Where("ledger = ?", store.ledger.Name), nil +} + +func (h logsResourceHandler) resolveFilter(_ *Store, _ ledgercontroller.ResourceQuery[any], operator, property string, value any) (string, []any, error) { + switch { + case property == "date": + return fmt.Sprintf("%s %s ?", property, convertOperatorToSQL(operator)), []any{value}, nil + default: + return "", nil, fmt.Errorf("unknown key '%s' when building query", property) + } +} + +func (h logsResourceHandler) expand(_ *Store, _ ledgercontroller.ResourceQuery[any], _ string) (*bun.SelectQuery, *joinCondition, error) { + return nil, nil, errors.New("no expand supported") +} + +func (h logsResourceHandler) project(store *Store, query ledgercontroller.ResourceQuery[any], selectQuery *bun.SelectQuery) (*bun.SelectQuery, error) { + return selectQuery.ColumnExpr("*"), nil +} + +var _ repositoryHandler[any] = logsResourceHandler{} diff --git a/internal/storage/ledger/resource_transactions.go b/internal/storage/ledger/resource_transactions.go new file mode 100644 index 000000000..8c94597aa --- /dev/null +++ b/internal/storage/ledger/resource_transactions.go @@ -0,0 +1,191 @@ +package ledger + +import ( + "fmt" + ledger "github.com/formancehq/ledger/internal" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/formancehq/ledger/pkg/features" + "github.com/uptrace/bun" + "slices" +) + +type transactionsResourceHandler struct{} + +func (h transactionsResourceHandler) filters() []filter { + return []filter{ + { + name: "reverted", + validators: []propertyValidator{ + acceptOperators("$match"), + }, + }, + { + name: "account", + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + return validateAddressFilter(l, operator, value) + }), + }, + }, + { + name: "source", + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + return validateAddressFilter(l, operator, value) + }), + }, + }, + { + name: "destination", + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + return validateAddressFilter(l, operator, value) + }), + }, + }, + { + // todo: add validators + name: "timestamp", + }, + { + name: "metadata", + validators: []propertyValidator{ + acceptOperators("$exists"), + }, + }, + { + name: `metadata\[.*]`, + validators: []propertyValidator{ + acceptOperators("$match"), + }, + }, + { + name: "id", + }, + } +} + +func (h transactionsResourceHandler) buildDataset(store *Store, opts repositoryHandlerBuildContext[any]) (*bun.SelectQuery, error) { + ret := store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName("transactions")). + Column( + "ledger", + "id", + "timestamp", + "reference", + "inserted_at", + "updated_at", + "postings", + "sources", + "destinations", + "sources_arrays", + "destinations_arrays", + ). + Where("ledger = ?", store.ledger.Name) + + if slices.Contains(opts.Expand, "volumes") { + ret = ret.Column("post_commit_volumes") + } + + if opts.PIT != nil && !opts.PIT.IsZero() { + ret = ret.Where("timestamp <= ?", opts.PIT) + } + + if store.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && opts.PIT != nil && !opts.PIT.IsZero() { + selectDistinctTransactionMetadataHistories := store.db.NewSelect(). + DistinctOn("transactions_id"). + ModelTableExpr(store.GetPrefixedRelationName("transactions_metadata")). + Where("ledger = ?", store.ledger.Name). + Column("transactions_id", "metadata"). + Order("transactions_id", "revision desc"). + Where("date <= ?", opts.PIT) + + ret = ret. + Join( + `left join (?) transactions_metadata on transactions_metadata.transactions_id = transactions.id`, + selectDistinctTransactionMetadataHistories, + ). + ColumnExpr("coalesce(transactions_metadata.metadata, '{}'::jsonb) as metadata") + } else { + ret = ret.ColumnExpr("metadata") + } + + if opts.UsePIT() { + ret = ret.ColumnExpr("(case when transactions.reverted_at <= ? then transactions.reverted_at else null end) as reverted_at", opts.PIT) + } else { + ret = ret.Column("reverted_at") + } + + return ret, nil +} + +func (h transactionsResourceHandler) resolveFilter(store *Store, opts ledgercontroller.ResourceQuery[any], operator, property string, value any) (string, []any, error) { + switch { + case property == "id": + return fmt.Sprintf("id %s ?", convertOperatorToSQL(operator)), []any{value}, nil + case property == "reference" || property == "timestamp": + return fmt.Sprintf("%s %s ?", property, convertOperatorToSQL(operator)), []any{value}, nil + case property == "reverted": + ret := "reverted_at is" + if value.(bool) { + ret += " not" + } + return ret + " null", nil, nil + case property == "account": + return filterAccountAddressOnTransactions(value.(string), true, true), nil, nil + case property == "source": + return filterAccountAddressOnTransactions(value.(string), true, false), nil, nil + case property == "destination": + return filterAccountAddressOnTransactions(value.(string), false, true), nil, nil + case metadataRegex.Match([]byte(property)): + match := metadataRegex.FindAllStringSubmatch(property, 3) + + return "metadata @> ?", []any{map[string]any{ + match[0][1]: value, + }}, nil + + case property == "metadata": + return "metadata -> ? is not null", []any{value}, nil + default: + return "", nil, fmt.Errorf("unsupported filter: %s", property) + } +} + +func (h transactionsResourceHandler) project(store *Store, query ledgercontroller.ResourceQuery[any], selectQuery *bun.SelectQuery) (*bun.SelectQuery, error) { + return selectQuery.ColumnExpr("*"), nil +} + +func (h transactionsResourceHandler) expand(store *Store, opts ledgercontroller.ResourceQuery[any], property string) (*bun.SelectQuery, *joinCondition, error) { + if property != "effectiveVolumes" { + return nil, nil, nil + } + + ret := store.db.NewSelect(). + TableExpr( + "(?) data", + store.db.NewSelect(). + TableExpr( + "(?) moves", + store.db.NewSelect(). + DistinctOn("transactions_id, accounts_address, asset"). + ModelTableExpr(store.GetPrefixedRelationName("moves")). + Column("transactions_id", "accounts_address", "asset"). + ColumnExpr(`first_value(moves.post_commit_effective_volumes) over (partition by (transactions_id, accounts_address, asset) order by seq desc) as post_commit_effective_volumes`). + Where("ledger = ?", store.ledger.Name). + Where("transactions_id in (select id from dataset)"), + ). + Column("transactions_id", "accounts_address"). + ColumnExpr(`public.aggregate_objects(json_build_object(moves.asset, json_build_object('input', (moves.post_commit_effective_volumes).inputs, 'output', (moves.post_commit_effective_volumes).outputs))::jsonb) AS post_commit_effective_volumes`). + Group("transactions_id", "accounts_address"), + ). + Column("transactions_id"). + ColumnExpr("public.aggregate_objects(json_build_object(accounts_address, post_commit_effective_volumes)::jsonb) AS post_commit_effective_volumes"). + Group("transactions_id") + + return ret, &joinCondition{ + left: "id", + right: "transactions_id", + }, nil +} + +var _ repositoryHandler[any] = transactionsResourceHandler{} diff --git a/internal/storage/ledger/resource_volumes.go b/internal/storage/ledger/resource_volumes.go new file mode 100644 index 000000000..c378c7f97 --- /dev/null +++ b/internal/storage/ledger/resource_volumes.go @@ -0,0 +1,216 @@ +package ledger + +import ( + "errors" + "fmt" + ledger "github.com/formancehq/ledger/internal" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/formancehq/ledger/pkg/features" + "github.com/uptrace/bun" + "strings" +) + +type volumesResourceHandler struct{} + +func (h volumesResourceHandler) filters() []filter { + return []filter{ + { + name: "address", + aliases: []string{"account"}, + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + return validateAddressFilter(l, operator, value) + }), + }, + }, + { + name: `balance(\[.*])?`, + validators: []propertyValidator{ + acceptOperators("$lt", "$gt", "$lte", "$gte", "$match"), + }, + }, + { + name: "metadata", + matchers: []func(string) bool{ + func(key string) bool { + return key == "metadata" || metadataRegex.Match([]byte(key)) + }, + }, + validators: []propertyValidator{ + propertyValidatorFunc(func(l ledger.Ledger, operator string, key string, value any) error { + if key == "metadata" { + if operator != "$exists" { + return fmt.Errorf("unsupported operator %s for metadata", operator) + } + return nil + } + if operator != "$match" { + return fmt.Errorf("unsupported operator %s for metadata", operator) + } + return nil + }), + }, + }, + } +} + +func (h volumesResourceHandler) buildDataset(store *Store, query repositoryHandlerBuildContext[ledgercontroller.GetVolumesOptions]) (*bun.SelectQuery, error) { + + var selectVolumes *bun.SelectQuery + + needAddressSegments := query.useFilter("address", isPartialAddress) + if !query.UsePIT() && !query.UseOOT() { + selectVolumes = store.db.NewSelect(). + Column("asset", "input", "output"). + ColumnExpr("input - output as balance"). + ColumnExpr("accounts_address as account"). + ModelTableExpr(store.GetPrefixedRelationName("accounts_volumes")). + Where("ledger = ?", store.ledger.Name). + Order("accounts_address", "asset") + + if query.useFilter("metadata") || needAddressSegments { + subQuery := store.db.NewSelect(). + TableExpr(store.GetPrefixedRelationName("accounts")). + Column("address"). + Where("ledger = ?", store.ledger.Name). + Where("accounts.address = accounts_address") + + if needAddressSegments { + subQuery = subQuery.ColumnExpr("address_array as account_array") + selectVolumes = selectVolumes.Column("account_array") + } + if query.useFilter("metadata") { + subQuery = subQuery.ColumnExpr("metadata") + selectVolumes = selectVolumes.Column("metadata") + } + + selectVolumes = selectVolumes. + Join(`join lateral (?) accounts on true`, subQuery) + } + } else { + if !store.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory) + } + + selectVolumes = store.db.NewSelect(). + Column("asset"). + ColumnExpr("accounts_address as account"). + ColumnExpr("sum(case when not is_source then amount else 0 end) as input"). + ColumnExpr("sum(case when is_source then amount else 0 end) as output"). + ColumnExpr("sum(case when not is_source then amount else -amount end) as balance"). + ModelTableExpr(store.GetPrefixedRelationName("moves")). + Where("ledger = ?", store.ledger.Name). + GroupExpr("accounts_address, asset"). + Order("accounts_address", "asset") + + dateFilterColumn := "effective_date" + if query.Opts.UseInsertionDate { + dateFilterColumn = "insertion_date" + } + + if query.UsePIT() { + selectVolumes = selectVolumes.Where(dateFilterColumn+" <= ?", query.PIT) + } + + if query.UseOOT() { + selectVolumes = selectVolumes.Where(dateFilterColumn+" >= ?", query.OOT) + } + + if needAddressSegments { + subQuery := store.db.NewSelect(). + TableExpr(store.GetPrefixedRelationName("accounts")). + Column("address_array"). + Where("accounts.address = accounts_address"). + Where("ledger = ?", store.ledger.Name) + + selectVolumes. + ColumnExpr("(array_agg(accounts.address_array))[1] as account_array"). + Join(`join lateral (?) accounts on true`, subQuery) + } + + if query.useFilter("metadata") { + subQuery := store.db.NewSelect(). + DistinctOn("accounts_address"). + ModelTableExpr(store.GetPrefixedRelationName("accounts_metadata")). + ColumnExpr("first_value(metadata) over (partition by accounts_address order by revision desc) as metadata"). + Where("ledger = ?", store.ledger.Name). + Where("accounts_metadata.accounts_address = moves.accounts_address"). + Where("date <= ?", query.PIT) + + selectVolumes = selectVolumes. + Join(`left join lateral (?) accounts_metadata on true`, subQuery). + ColumnExpr("(array_agg(metadata))[1] as metadata") + } + } + + return selectVolumes, nil +} + +func (h volumesResourceHandler) resolveFilter( + store *Store, + opts ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions], + operator, property string, + value any, +) (string, []any, error) { + + switch { + case property == "address" || property == "account": + return filterAccountAddress(value.(string), "account"), nil, nil + case balanceRegex.MatchString(property) || property == "balance": + clauses := make([]string, 0) + args := make([]any, 0) + + clauses = append(clauses, "balance "+convertOperatorToSQL(operator)+" ?") + args = append(args, value) + + if balanceRegex.MatchString(property) { + clauses = append(clauses, "asset = ?") + args = append(args, balanceRegex.FindAllStringSubmatch(property, 2)[0][1]) + } + + return "(" + strings.Join(clauses, ") and (") + ")", args, nil + case metadataRegex.Match([]byte(property)) || property == "metadata": + if property == "metadata" { + return "metadata -> ? is not null", []any{value}, nil + } else { + match := metadataRegex.FindAllStringSubmatch(property, 3) + + return "metadata @> ?", []any{map[string]any{ + match[0][1]: value, + }}, nil + } + default: + return "", nil, fmt.Errorf("unsupported filter %s", property) + } +} + +func (h volumesResourceHandler) project( + store *Store, + query ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions], + selectQuery *bun.SelectQuery, +) (*bun.SelectQuery, error) { + selectQuery = selectQuery.DistinctOn("account, asset") + + if query.Opts.GroupLvl == 0 { + return selectQuery.ColumnExpr("*"), nil + } + + intermediate := store.db.NewSelect(). + ModelTableExpr("(?) data", selectQuery). + Column("asset", "input", "output", "balance"). + ColumnExpr(fmt.Sprintf(`(array_to_string((string_to_array(account, ':'))[1:LEAST(array_length(string_to_array(account, ':'),1),%d)],':')) as account`, query.Opts.GroupLvl)) + + return store.db.NewSelect(). + ModelTableExpr("(?) data", intermediate). + Column("account", "asset"). + ColumnExpr("sum(input) as input"). + ColumnExpr("sum(output) as output"). + ColumnExpr("sum(balance) as balance"). + GroupExpr("account, asset"), nil +} + +func (h volumesResourceHandler) expand(_ *Store, _ ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions], property string) (*bun.SelectQuery, *joinCondition, error) { + return nil, nil, errors.New("no expansion available") +} + +var _ repositoryHandler[ledgercontroller.GetVolumesOptions] = volumesResourceHandler{} diff --git a/internal/storage/ledger/store.go b/internal/storage/ledger/store.go index 34a2b8809..f847f0dff 100644 --- a/internal/storage/ledger/store.go +++ b/internal/storage/ledger/store.go @@ -4,8 +4,10 @@ import ( "context" "database/sql" "fmt" + "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/migrations" "github.com/formancehq/go-libs/v2/platform/postgres" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/ledger/internal/storage/bucket" "github.com/formancehq/ledger/pkg/features" "go.opentelemetry.io/otel/metric" @@ -25,43 +27,80 @@ type Store struct { tracer trace.Tracer meter metric.Meter - listAccountsHistogram metric.Int64Histogram checkBucketSchemaHistogram metric.Int64Histogram checkLedgerSchemaHistogram metric.Int64Histogram - getAccountHistogram metric.Int64Histogram - countAccountsHistogram metric.Int64Histogram updateAccountsMetadataHistogram metric.Int64Histogram - deleteAccountMetadataHistogram metric.Int64Histogram - upsertAccountsHistogram metric.Int64Histogram - getBalancesHistogram metric.Int64Histogram + deleteAccountMetadataHistogram metric.Int64Histogram + upsertAccountsHistogram metric.Int64Histogram + getBalancesHistogram metric.Int64Histogram insertLogHistogram metric.Int64Histogram - listLogsHistogram metric.Int64Histogram readLogWithIdempotencyKeyHistogram metric.Int64Histogram insertMovesHistogram metric.Int64Histogram - countTransactionsHistogram metric.Int64Histogram - getTransactionHistogram metric.Int64Histogram insertTransactionHistogram metric.Int64Histogram revertTransactionHistogram metric.Int64Histogram updateTransactionMetadataHistogram metric.Int64Histogram deleteTransactionMetadataHistogram metric.Int64Histogram updateBalancesHistogram metric.Int64Histogram getVolumesWithBalancesHistogram metric.Int64Histogram - listTransactionsHistogram metric.Int64Histogram } -func (s *Store) BeginTX(ctx context.Context, options *sql.TxOptions) (*Store, error) { - tx, err := s.db.BeginTx(ctx, options) +func (store *Store) Volumes() ledgercontroller.PaginatedResource[ + ledger.VolumesWithBalanceByAssetByAccount, + ledgercontroller.GetVolumesOptions, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]] { + return newPaginatedResourceRepository(store, store.ledger, &volumesResourceHandler{}, offsetPaginator[ledger.VolumesWithBalanceByAssetByAccount, ledgercontroller.GetVolumesOptions]{ + defaultPaginationColumn: "account", + defaultOrder: bunpaginate.OrderAsc, + }) +} + +func (store *Store) AggregatedVolumes() ledgercontroller.Resource[ledger.AggregatedVolumes, ledgercontroller.GetAggregatedVolumesOptions] { + return newResourceRepository[ledger.AggregatedVolumes, ledgercontroller.GetAggregatedVolumesOptions](store, store.ledger, &aggregatedBalancesResourceRepositoryHandler{}) +} + +func (store *Store) Transactions() ledgercontroller.PaginatedResource[ + ledger.Transaction, + any, + ledgercontroller.ColumnPaginatedQuery[any]] { + return newPaginatedResourceRepository(store, store.ledger, &transactionsResourceHandler{}, columnPaginator[ledger.Transaction, any]{ + defaultPaginationColumn: "id", + defaultOrder: bunpaginate.OrderDesc, + }) +} + +func (store *Store) Logs() ledgercontroller.PaginatedResource[ + ledger.Log, + any, + ledgercontroller.ColumnPaginatedQuery[any]] { + return newPaginatedResourceRepositoryMapper[ledger.Log, Log, any, ledgercontroller.ColumnPaginatedQuery[any]](store, store.ledger, &logsResourceHandler{}, columnPaginator[Log, any]{ + defaultPaginationColumn: "id", + defaultOrder: bunpaginate.OrderDesc, + }) +} + +func (store *Store) Accounts() ledgercontroller.PaginatedResource[ + ledger.Account, + any, + ledgercontroller.OffsetPaginatedQuery[any]] { + return newPaginatedResourceRepository(store, store.ledger, &accountsResourceHandler{}, offsetPaginator[ledger.Account, any]{ + defaultPaginationColumn: "address", + defaultOrder: bunpaginate.OrderAsc, + }) +} + +func (store *Store) BeginTX(ctx context.Context, options *sql.TxOptions) (*Store, error) { + tx, err := store.db.BeginTx(ctx, options) if err != nil { return nil, postgres.ResolveError(err) } - cp := *s + cp := *store cp.db = tx return &cp, nil } -func (s *Store) Commit() error { - switch db := s.db.(type) { +func (store *Store) Commit() error { + switch db := store.db.(type) { case bun.Tx: return db.Commit() default: @@ -69,8 +108,8 @@ func (s *Store) Commit() error { } } -func (s *Store) Rollback() error { - switch db := s.db.(type) { +func (store *Store) Rollback() error { + switch db := store.db.(type) { case bun.Tx: return db.Rollback() default: @@ -78,44 +117,44 @@ func (s *Store) Rollback() error { } } -func (s *Store) GetLedger() ledger.Ledger { - return s.ledger +func (store *Store) GetLedger() ledger.Ledger { + return store.ledger } -func (s *Store) GetDB() bun.IDB { - return s.db +func (store *Store) GetDB() bun.IDB { + return store.db } -func (s *Store) GetBucket() bucket.Bucket { - return s.bucket +func (store *Store) GetBucket() bucket.Bucket { + return store.bucket } -func (s *Store) GetPrefixedRelationName(v string) string { - return fmt.Sprintf(`"%s".%s`, s.ledger.Bucket, v) +func (store *Store) GetPrefixedRelationName(v string) string { + return fmt.Sprintf(`"%s".%s`, store.ledger.Bucket, v) } -func (s *Store) validateAddressFilter(operator string, value any) error { +func validateAddressFilter(ledger ledger.Ledger, operator string, value any) error { if operator != "$match" { - return errors.New("'address' column can only be used with $match") + return fmt.Errorf("'address' column can only be used with $match, operator used is: %s", operator) } if value, ok := value.(string); !ok { return fmt.Errorf("invalid 'address' filter") - } else if isSegmentedAddress(value) && !s.ledger.HasFeature(features.FeatureIndexAddressSegments, "ON") { + } else if isSegmentedAddress(value) && !ledger.HasFeature(features.FeatureIndexAddressSegments, "ON") { return fmt.Errorf("feature %s must be 'ON' to use segments address", features.FeatureIndexAddressSegments) } return nil } -func (s *Store) LockLedger(ctx context.Context) error { - _, err := s.db.NewRaw(`lock table ` + s.GetPrefixedRelationName("logs")).Exec(ctx) +func (store *Store) LockLedger(ctx context.Context) error { + _, err := store.db.NewRaw(`lock table ` + store.GetPrefixedRelationName("logs")).Exec(ctx) return postgres.ResolveError(err) } -func New(db bun.IDB, bucket bucket.Bucket, ledger ledger.Ledger, opts ...Option) *Store { +func New(db bun.IDB, bucket bucket.Bucket, l ledger.Ledger, opts ...Option) *Store { ret := &Store{ db: db, - ledger: ledger, + ledger: l, bucket: bucket, } for _, opt := range append(defaultOptions, opts...) { @@ -123,11 +162,6 @@ func New(db bun.IDB, bucket bucket.Bucket, ledger ledger.Ledger, opts ...Option) } var err error - ret.listAccountsHistogram, err = ret.meter.Int64Histogram("store.listAccounts") - if err != nil { - panic(err) - } - ret.checkBucketSchemaHistogram, err = ret.meter.Int64Histogram("store.checkBucketSchema") if err != nil { panic(err) @@ -138,16 +172,6 @@ func New(db bun.IDB, bucket bucket.Bucket, ledger ledger.Ledger, opts ...Option) panic(err) } - ret.getAccountHistogram, err = ret.meter.Int64Histogram("store.getAccount") - if err != nil { - panic(err) - } - - ret.countAccountsHistogram, err = ret.meter.Int64Histogram("store.countAccounts") - if err != nil { - panic(err) - } - ret.updateAccountsMetadataHistogram, err = ret.meter.Int64Histogram("store.updateAccountsMetadata") if err != nil { panic(err) @@ -173,11 +197,6 @@ func New(db bun.IDB, bucket bucket.Bucket, ledger ledger.Ledger, opts ...Option) panic(err) } - ret.listLogsHistogram, err = ret.meter.Int64Histogram("store.listLogs") - if err != nil { - panic(err) - } - ret.readLogWithIdempotencyKeyHistogram, err = ret.meter.Int64Histogram("store.readLogWithIdempotencyKey") if err != nil { panic(err) @@ -188,16 +207,6 @@ func New(db bun.IDB, bucket bucket.Bucket, ledger ledger.Ledger, opts ...Option) panic(err) } - ret.countTransactionsHistogram, err = ret.meter.Int64Histogram("store.countTransactions") - if err != nil { - panic(err) - } - - ret.getTransactionHistogram, err = ret.meter.Int64Histogram("store.getTransaction") - if err != nil { - panic(err) - } - ret.insertTransactionHistogram, err = ret.meter.Int64Histogram("store.insertTransaction") if err != nil { panic(err) @@ -228,24 +237,19 @@ func New(db bun.IDB, bucket bucket.Bucket, ledger ledger.Ledger, opts ...Option) panic(err) } - ret.listTransactionsHistogram, err = ret.meter.Int64Histogram("store.listTransactions") - if err != nil { - panic(err) - } - return ret } -func (s *Store) HasMinimalVersion(ctx context.Context) (bool, error) { - return s.bucket.HasMinimalVersion(ctx) +func (store *Store) HasMinimalVersion(ctx context.Context) (bool, error) { + return store.bucket.HasMinimalVersion(ctx) } -func (s *Store) GetMigrationsInfo(ctx context.Context) ([]migrations.Info, error) { - return s.bucket.GetMigrationsInfo(ctx) +func (store *Store) GetMigrationsInfo(ctx context.Context) ([]migrations.Info, error) { + return store.bucket.GetMigrationsInfo(ctx) } -func (s *Store) WithDB(db bun.IDB) *Store { - ret := *s +func (store *Store) WithDB(db bun.IDB) *Store { + ret := *store ret.db = db return &ret diff --git a/internal/storage/ledger/transactions.go b/internal/storage/ledger/transactions.go index 9de7d93fd..58d461091 100644 --- a/internal/storage/ledger/transactions.go +++ b/internal/storage/ledger/transactions.go @@ -26,7 +26,6 @@ import ( "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/metadata" - "github.com/formancehq/go-libs/v2/query" ledger "github.com/formancehq/ledger/internal" "github.com/uptrace/bun" ) @@ -35,240 +34,36 @@ var ( metadataRegex = regexp.MustCompile(`metadata\[(.+)]`) ) -func (s *Store) selectDistinctTransactionMetadataHistories(date *time.Time) *bun.SelectQuery { - ret := s.db.NewSelect(). - DistinctOn("transactions_id"). - ModelTableExpr(s.GetPrefixedRelationName("transactions_metadata")). - Where("ledger = ?", s.ledger.Name). - Column("transactions_id", "metadata"). - Order("transactions_id", "revision desc") +func (store *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) error { - if date != nil && !date.IsZero() { - ret = ret.Where("date <= ?", date) - } - - return ret -} - -func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffectiveVolumes bool, q query.Builder) (*bun.SelectQuery, error) { - - ret := s.db.NewSelect() - if expandEffectiveVolumes && !s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { - return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes) - } - - if q != nil { - if err := q.Walk(func(operator, key string, value any) error { - switch { - case key == "reverted": - if operator != "$match" { - return ledgercontroller.NewErrInvalidQuery("'reverted' column can only be used with $match") - } - switch value.(type) { - case bool: - return nil - default: - return ledgercontroller.NewErrInvalidQuery("'reverted' can only be used with bool value") - } - case key == "account": - return s.validateAddressFilter(operator, value) - case key == "source": - return s.validateAddressFilter(operator, value) - case key == "destination": - return s.validateAddressFilter(operator, value) - case key == "timestamp": - case metadataRegex.Match([]byte(key)): - if operator != "$match" { - return ledgercontroller.NewErrInvalidQuery("'metadata[xxx]' column can only be used with $match") - } - case key == "metadata": - if operator != "$exists" { - return ledgercontroller.NewErrInvalidQuery("'metadata' key filter can only be used with $exists") - } - default: - return ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) - } - - return nil - }); err != nil { - return nil, err - } - } - - ret = ret. - ModelTableExpr(s.GetPrefixedRelationName("transactions")). - Column( - "ledger", - "id", - "timestamp", - "reference", - "inserted_at", - "updated_at", - "postings", - "sources", - "destinations", - "sources_arrays", - "destinations_arrays", - "reverted_at", - "post_commit_volumes", - ). - Where("ledger = ?", s.ledger.Name) - - if date != nil && !date.IsZero() { - ret = ret.Where("timestamp <= ?", date) - } - - if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { - ret = ret. - Join( - `left join (?) transactions_metadata on transactions_metadata.transactions_id = transactions.id`, - s.selectDistinctTransactionMetadataHistories(date), - ). - ColumnExpr("coalesce(transactions_metadata.metadata, '{}'::jsonb) as metadata") - } else { - ret = ret.ColumnExpr("metadata") - } - - if s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") && expandEffectiveVolumes { - ret = ret. - Join( - `join (?) pcev on pcev.transactions_id = transactions.id`, - s.db.NewSelect(). - TableExpr( - "(?) data", - s.db.NewSelect(). - TableExpr( - "(?) moves", - s.db.NewSelect(). - DistinctOn("transactions_id, accounts_address, asset"). - ModelTableExpr(s.GetPrefixedRelationName("moves")). - Column("transactions_id", "accounts_address", "asset"). - Where("ledger = ?", s.ledger.Name). - ColumnExpr(`first_value(moves.post_commit_effective_volumes) over (partition by (transactions_id, accounts_address, asset) order by seq desc) as post_commit_effective_volumes`), - ). - Column("transactions_id"). - ColumnExpr(` - json_build_object( - moves.accounts_address, - json_build_object( - moves.asset, - json_build_object( - 'input', (moves.post_commit_effective_volumes).inputs, - 'output', (moves.post_commit_effective_volumes).outputs - ) - ) - ) as post_commit_effective_volumes - `), - ). - Column("transactions_id"). - ColumnExpr("public.aggregate_objects(post_commit_effective_volumes::jsonb) as post_commit_effective_volumes"). - Group("transactions_id"), - ). - ColumnExpr("pcev.*") - } - - // Create a parent query which set reverted_at to null if the date passed as argument is before - ret = s.db.NewSelect(). - ModelTableExpr("(?) transactions", ret). - Column( - "ledger", - "id", - "timestamp", - "reference", - "inserted_at", - "updated_at", - "postings", - "sources", - "destinations", - "sources_arrays", - "destinations_arrays", - "metadata", - ) - if expandVolumes { - ret = ret.Column("post_commit_volumes") - } - if expandEffectiveVolumes { - ret = ret.Column("post_commit_effective_volumes") - } - if date != nil && !date.IsZero() { - ret = ret.ColumnExpr("(case when transactions.reverted_at <= ? then transactions.reverted_at else null end) as reverted_at", date) - } else { - ret = ret.Column("reverted_at") - } - - if q != nil { - where, args, err := q.Build(query.ContextFn(func(key, operator string, value any) (string, []any, error) { - switch { - case key == "reference" || key == "timestamp": - return fmt.Sprintf("%s %s ?", key, query.DefaultComparisonOperatorsMapping[operator]), []any{value}, nil - case key == "reverted": - ret := "reverted_at is" - if value.(bool) { - ret += " not" - } - return ret + " null", nil, nil - case key == "account": - return filterAccountAddressOnTransactions(value.(string), true, true), nil, nil - case key == "source": - return filterAccountAddressOnTransactions(value.(string), true, false), nil, nil - case key == "destination": - return filterAccountAddressOnTransactions(value.(string), false, true), nil, nil - case metadataRegex.Match([]byte(key)): - match := metadataRegex.FindAllStringSubmatch(key, 3) - - return "metadata @> ?", []any{map[string]any{ - match[0][1]: value, - }}, nil - - case key == "metadata": - return "metadata -> ? is not null", []any{value}, nil - case key == "timestamp": - return fmt.Sprintf("timestamp %s ?", convertOperatorToSQL(operator)), []any{value}, nil - default: - return "", nil, ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) - } - })) - if err != nil { - return nil, err - } - - if len(args) > 0 { - ret = ret.Where(where, args...) - } else { - ret = ret.Where(where) - } - } - - return ret, nil -} - -func (s *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) error { - - postCommitVolumes, err := s.UpdateVolumes(ctx, tx.VolumeUpdates()...) + postCommitVolumes, err := store.UpdateVolumes(ctx, tx.VolumeUpdates()...) if err != nil { return fmt.Errorf("failed to update balances: %w", err) } tx.PostCommitVolumes = postCommitVolumes.Copy() - err = s.InsertTransaction(ctx, tx) + err = store.InsertTransaction(ctx, tx) if err != nil { return fmt.Errorf("failed to insert transaction: %w", err) } - err = s.UpsertAccounts(ctx, collectionutils.Map(tx.InvolvedAccounts(), func(address string) *ledger.Account { + err = store.UpsertAccounts(ctx, collectionutils.Map(tx.InvolvedAccounts(), func(address string) *ledger.Account { return &ledger.Account{ - Address: address, - FirstUsage: tx.Timestamp, - Metadata: make(metadata.Metadata), + Address: address, + FirstUsage: tx.Timestamp, + Metadata: make(metadata.Metadata), + InsertionDate: tx.InsertedAt, + UpdatedAt: tx.InsertedAt, } })...) if err != nil { return fmt.Errorf("upserting accounts: %w", err) } - if s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + if store.ledger.HasFeature(features.FeatureMovesHistory, "ON") { moves := ledger.Moves{} - postings := tx.Postings + postings := make([]ledger.Posting, len(tx.Postings)) + copy(postings, tx.Postings) slices.Reverse(postings) for _, posting := range postings { @@ -298,11 +93,11 @@ func (s *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) e slices.Reverse(moves) - if err := s.InsertMoves(ctx, moves...); err != nil { + if err := store.InsertMoves(ctx, moves...); err != nil { return fmt.Errorf("failed to insert moves: %w", err) } - if s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { + if store.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { tx.PostCommitEffectiveVolumes = moves.ComputePostCommitEffectiveVolumes() } } @@ -310,105 +105,21 @@ func (s *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) e return nil } -func (s *Store) ListTransactions(ctx context.Context, q ledgercontroller.ListTransactionsQuery) (*bunpaginate.Cursor[ledger.Transaction], error) { - return tracing.TraceWithMetric( - ctx, - "ListTransactions", - s.tracer, - s.listTransactionsHistogram, - func(ctx context.Context) (*bunpaginate.Cursor[ledger.Transaction], error) { - selectTransactions, err := s.selectTransactions( - q.Options.Options.PIT, - q.Options.Options.ExpandVolumes, - q.Options.Options.ExpandEffectiveVolumes, - q.Options.QueryBuilder, - ) - if err != nil { - return nil, err - } - cursor, err := bunpaginate.UsingColumn[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes], ledger.Transaction]( - ctx, - selectTransactions, - bunpaginate.ColumnPaginatedQuery[ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes]](q), - ) - if err != nil { - return nil, err - } - - return cursor, nil - }, - ) -} - -func (s *Store) CountTransactions(ctx context.Context, q ledgercontroller.ListTransactionsQuery) (int, error) { - return tracing.TraceWithMetric( - ctx, - "CountTransactions", - s.tracer, - s.countTransactionsHistogram, - func(ctx context.Context) (int, error) { - selectTransactions, err := s.selectTransactions( - q.Options.Options.PIT, - q.Options.Options.ExpandVolumes, - q.Options.Options.ExpandEffectiveVolumes, - q.Options.QueryBuilder, - ) - if err != nil { - return 0, err - } - return s.db.NewSelect(). - TableExpr("(?) data", selectTransactions). - Count(ctx) - }, - ) -} - -func (s *Store) GetTransaction(ctx context.Context, filter ledgercontroller.GetTransactionQuery) (*ledger.Transaction, error) { - return tracing.TraceWithMetric( - ctx, - "GetTransaction", - s.tracer, - s.getTransactionHistogram, - func(ctx context.Context) (*ledger.Transaction, error) { - - ret := &ledger.Transaction{} - selectTransactions, err := s.selectTransactions( - filter.PIT, - filter.ExpandVolumes, - filter.ExpandEffectiveVolumes, - nil, - ) - if err != nil { - return nil, err - } - if err := selectTransactions. - Where("transactions.id = ?", filter.ID). - Limit(1). - Model(ret). - Scan(ctx); err != nil { - return nil, postgres.ResolveError(err) - } - - return ret, nil - }, - ) -} - -func (s *Store) InsertTransaction(ctx context.Context, tx *ledger.Transaction) error { - _, err := tracing.TraceWithMetric( +func (store *Store) InsertTransaction(ctx context.Context, tx *ledger.Transaction) error { + return tracing.SkipResult(tracing.TraceWithMetric( ctx, "InsertTransaction", - s.tracer, - s.insertTransactionHistogram, + store.tracer, + store.insertTransactionHistogram, func(ctx context.Context) (*ledger.Transaction, error) { - query := s.db.NewInsert(). + query := store.db.NewInsert(). Model(tx). - ModelTableExpr(s.GetPrefixedRelationName("transactions")). - Value("ledger", "?", s.ledger.Name). + ModelTableExpr(store.GetPrefixedRelationName("transactions")). + Value("ledger", "?", store.ledger.Name). Returning("id, timestamp, inserted_at") if tx.ID == 0 { - query = query.Value("id", "nextval(?)", s.GetPrefixedRelationName(fmt.Sprintf(`"transaction_id_%d"`, s.ledger.ID))) + query = query.Value("id", "nextval(?)", store.GetPrefixedRelationName(fmt.Sprintf(`"transaction_id_%d"`, store.ledger.ID))) } _, err := query.Exec(ctx) @@ -432,31 +143,29 @@ func (s *Store) InsertTransaction(ctx context.Context, tx *ledger.Transaction) e attribute.String("timestamp", tx.Timestamp.Format(time.RFC3339Nano)), ) }, - ) - - return err + )) } // updateTxWithRetrieve try to apply to provided update query and check (if the update return no rows modified), that the row exists -func (s *Store) updateTxWithRetrieve(ctx context.Context, id int, query *bun.UpdateQuery) (*ledger.Transaction, bool, error) { +func (store *Store) updateTxWithRetrieve(ctx context.Context, id int, query *bun.UpdateQuery) (*ledger.Transaction, bool, error) { type modifiedEntity struct { ledger.Transaction `bun:",extend"` Modified bool `bun:"modified"` } me := &modifiedEntity{} - err := s.db.NewSelect(). + err := store.db.NewSelect(). With("upd", query). ModelTableExpr( "(?) transactions", - s.db.NewSelect(). + store.db.NewSelect(). ColumnExpr("upd.*, true as modified"). ModelTableExpr("upd"). UnionAll( - s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("transactions")). + store.db.NewSelect(). + ModelTableExpr(store.GetPrefixedRelationName("transactions")). ColumnExpr("*, false as modified"). - Where("id = ? and ledger = ?", id, s.ledger.Name). + Where("id = ? and ledger = ?", id, store.ledger.Name). Limit(1), ), ). @@ -464,26 +173,23 @@ func (s *Store) updateTxWithRetrieve(ctx context.Context, id int, query *bun.Upd ColumnExpr("*"). Limit(1). Scan(ctx) - if err != nil { - return nil, false, postgres.ResolveError(err) - } - return &me.Transaction, me.Modified, nil + return &me.Transaction, me.Modified, postgres.ResolveError(err) } -func (s *Store) RevertTransaction(ctx context.Context, id int, at time.Time) (tx *ledger.Transaction, modified bool, err error) { +func (store *Store) RevertTransaction(ctx context.Context, id int, at time.Time) (tx *ledger.Transaction, modified bool, err error) { _, err = tracing.TraceWithMetric( ctx, "RevertTransaction", - s.tracer, - s.revertTransactionHistogram, + store.tracer, + store.revertTransactionHistogram, func(ctx context.Context) (*ledger.Transaction, error) { - query := s.db.NewUpdate(). + query := store.db.NewUpdate(). Model(&ledger.Transaction{}). - ModelTableExpr(s.GetPrefixedRelationName("transactions")). + ModelTableExpr(store.GetPrefixedRelationName("transactions")). Where("id = ?", id). Where("reverted_at is null"). - Where("ledger = ?", s.ledger.Name). + Where("ledger = ?", store.ledger.Name). Returning("*") if at.IsZero() { query = query. @@ -495,71 +201,62 @@ func (s *Store) RevertTransaction(ctx context.Context, id int, at time.Time) (tx Set("updated_at = ?", at) } - tx, modified, err = s.updateTxWithRetrieve(ctx, id, query) + tx, modified, err = store.updateTxWithRetrieve(ctx, id, query) return nil, err }, ) - if err != nil { - return nil, false, err - } return tx, modified, err } -func (s *Store) UpdateTransactionMetadata(ctx context.Context, id int, m metadata.Metadata) (tx *ledger.Transaction, modified bool, err error) { +func (store *Store) UpdateTransactionMetadata(ctx context.Context, id int, m metadata.Metadata) (tx *ledger.Transaction, modified bool, err error) { _, err = tracing.TraceWithMetric( ctx, "UpdateTransactionMetadata", - s.tracer, - s.updateTransactionMetadataHistogram, + store.tracer, + store.updateTransactionMetadataHistogram, func(ctx context.Context) (*ledger.Transaction, error) { - tx, modified, err = s.updateTxWithRetrieve( + tx, modified, err = store.updateTxWithRetrieve( ctx, id, - s.db.NewUpdate(). + store.db.NewUpdate(). Model(&ledger.Transaction{}). - ModelTableExpr(s.GetPrefixedRelationName("transactions")). + ModelTableExpr(store.GetPrefixedRelationName("transactions")). Where("id = ?", id). - Where("ledger = ?", s.ledger.Name). + Where("ledger = ?", store.ledger.Name). Set("metadata = metadata || ?", m). Set("updated_at = (now() at time zone 'utc')"). Where("not (metadata @> ?)", m). Returning("*"), ) - return nil, err + return nil, postgres.ResolveError(err) }, ) - if err != nil { - return nil, false, err - } return tx, modified, err } -func (s *Store) DeleteTransactionMetadata(ctx context.Context, id int, key string) (tx *ledger.Transaction, modified bool, err error) { +func (store *Store) DeleteTransactionMetadata(ctx context.Context, id int, key string) (tx *ledger.Transaction, modified bool, err error) { _, err = tracing.TraceWithMetric( ctx, "DeleteTransactionMetadata", - s.tracer, - s.deleteTransactionMetadataHistogram, + store.tracer, + store.deleteTransactionMetadataHistogram, func(ctx context.Context) (*ledger.Transaction, error) { - tx, modified, err = s.updateTxWithRetrieve( + tx, modified, err = store.updateTxWithRetrieve( ctx, id, - s.db.NewUpdate(). + store.db.NewUpdate(). Model(&ledger.Transaction{}). - ModelTableExpr(s.GetPrefixedRelationName("transactions")). + ModelTableExpr(store.GetPrefixedRelationName("transactions")). Set("metadata = metadata - ?", key). Set("updated_at = (now() at time zone 'utc')"). Where("id = ?", id). - Where("ledger = ?", s.ledger.Name). + Where("ledger = ?", store.ledger.Name). Where("metadata -> ? is not null", key). Returning("*"), ) - return nil, err + return nil, postgres.ResolveError(err) }, ) - if err != nil { - return nil, false, err - } return tx, modified, err } diff --git a/internal/storage/ledger/transactions_test.go b/internal/storage/ledger/transactions_test.go index 32702020c..f7442370b 100644 --- a/internal/storage/ledger/transactions_test.go +++ b/internal/storage/ledger/transactions_test.go @@ -52,9 +52,10 @@ func TestTransactionsGetWithVolumes(t *testing.T) { err = store.CommitTransaction(ctx, &tx2) require.NoError(t, err) - tx, err := store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx1.ID). - WithExpandVolumes(). - WithExpandEffectiveVolumes()) + tx, err := store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx1.ID), + Expand: []string{"volumes", "effectiveVolumes"}, + }) require.NoError(t, err) require.Equal(t, tx1.Postings, tx.Postings) require.Equal(t, tx1.Reference, tx.Reference) @@ -75,9 +76,10 @@ func TestTransactionsGetWithVolumes(t *testing.T) { }, }, tx.PostCommitVolumes) - tx, err = store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx2.ID). - WithExpandVolumes(). - WithExpandEffectiveVolumes()) + tx, err = store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx2.ID), + Expand: []string{"volumes", "effectiveVolumes"}, + }) require.NoError(t, err) require.Equal(t, tx2.Postings, tx.Postings) require.Equal(t, tx2.Reference, tx.Reference) @@ -111,7 +113,7 @@ func TestTransactionsCount(t *testing.T) { require.NoError(t, err) } - count, err := store.CountTransactions(ctx, ledgercontroller.NewListTransactionsQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}))) + count, err := store.Transactions().Count(ctx, ledgercontroller.ResourceQuery[any]{}) require.NoError(t, err, "counting transactions should not fail") require.Equal(t, 3, count, "count should be equal") } @@ -148,11 +150,17 @@ func TestTransactionUpdateMetadata(t *testing.T) { require.NoError(t, err) // Check that the database returns metadata - tx, err := store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx1.ID).WithExpandVolumes().WithExpandEffectiveVolumes()) + tx, err := store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx1.ID), + Expand: []string{"volumes", "effectiveVolumes"}, + }) require.NoError(t, err, "getting transaction should not fail") require.Equal(t, tx.Metadata, metadata.Metadata{"foo1": "bar2"}, "metadata should be equal") - tx, err = store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx2.ID).WithExpandVolumes().WithExpandEffectiveVolumes()) + tx, err = store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx2.ID), + Expand: []string{"volumes", "effectiveVolumes"}, + }) require.NoError(t, err, "getting transaction should not fail") require.Equal(t, tx.Metadata, metadata.Metadata{"foo2": "bar2"}, "metadata should be equal") @@ -185,7 +193,9 @@ func TestTransactionDeleteMetadata(t *testing.T) { require.NoError(t, err) // Get from database and check metadata presence - tx, err := store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx1.ID)) + tx, err := store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx1.ID), + }) require.NoError(t, err) require.Equal(t, tx.Metadata, metadata.Metadata{"foo1": "bar1", "foo2": "bar2"}) @@ -194,7 +204,9 @@ func TestTransactionDeleteMetadata(t *testing.T) { require.NoError(t, err) require.True(t, modified) - tx, err = store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx1.ID)) + tx, err = store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx1.ID), + }) require.NoError(t, err) require.Equal(t, metadata.Metadata{"foo2": "bar2"}, tx.Metadata) @@ -437,12 +449,12 @@ func TestTransactionsCommit(t *testing.T) { require.NoError(t, err) } - cursor, err := store.ListTransactions(ctx, ledgercontroller.NewListTransactionsQuery( - ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - ExpandVolumes: true, - }). - WithPageSize(countTx)), - ) + cursor, err := store.Transactions().Paginate(ctx, ledgercontroller.ColumnPaginatedQuery[any]{ + PageSize: countTx, + Options: ledgercontroller.ResourceQuery[any]{ + Expand: []string{"volumes"}, + }, + }) require.NoError(t, err) require.Len(t, cursor.Data, countTx) @@ -506,7 +518,10 @@ func TestInsertTransactionInPast(t *testing.T) { err = store.CommitTransaction(ctx, &tx4) require.NoError(t, err) - tx2FromDatabase, err := store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx2.ID).WithExpandVolumes().WithExpandEffectiveVolumes()) + tx2FromDatabase, err := store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx2.ID), + Expand: []string{"volumes", "effectiveVolumes"}, + }) require.NoError(t, err) RequireEqual(t, ledger.PostCommitVolumes{ @@ -518,7 +533,9 @@ func TestInsertTransactionInPast(t *testing.T) { }, }, tx2FromDatabase.PostCommitEffectiveVolumes) - account, err := store.GetAccount(ctx, ledgercontroller.NewGetAccountQuery("bank")) + account, err := store.Accounts().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("address", "bank"), + }) require.NoError(t, err) require.Equal(t, tx4.Timestamp, account.FirstUsage) } @@ -558,10 +575,9 @@ func TestTransactionsRevert(t *testing.T) { require.False(t, reverted) // Revert a not existing transaction - revertedTx, reverted, err = store.RevertTransaction(ctx, 2, time.Time{}) + _, reverted, err = store.RevertTransaction(ctx, 2, time.Time{}) require.True(t, errors.Is(err, postgres.ErrNotFound)) require.False(t, reverted) - require.Nil(t, revertedTx) } func TestTransactionsInsert(t *testing.T) { @@ -660,6 +676,7 @@ func TestTransactionsList(t *testing.T) { tx1 := ledger.NewTransaction(). WithPostings( ledger.NewPosting("world", "alice", "USD", big.NewInt(100)), + ledger.NewPosting("world", "alice", "EUR", big.NewInt(100)), ). WithMetadata(metadata.Metadata{"category": "1"}). WithTimestamp(now.Add(-3 * time.Hour)) @@ -700,9 +717,10 @@ func TestTransactionsList(t *testing.T) { // refresh tx3 // we can't take the result of the call on RevertTransaction nor UpdateTransactionMetadata as the result does not contains pc(e)v tx3 := func() ledger.Transaction { - tx3, err := store.GetTransaction(ctx, ledgercontroller.NewGetTransactionQuery(tx3BeforeRevert.ID). - WithExpandVolumes(). - WithExpandEffectiveVolumes()) + tx3, err := store.Transactions().GetOne(ctx, ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("id", tx3BeforeRevert.ID), + Expand: []string{"volumes", "effectiveVolumes"}, + }) require.NoError(t, err) return *tx3 }() @@ -717,87 +735,114 @@ func TestTransactionsList(t *testing.T) { type testCase struct { name string - query ledgercontroller.PaginatedQueryOptions[ledgercontroller.PITFilterWithVolumes] + query ledgercontroller.ColumnPaginatedQuery[any] expected []ledger.Transaction expectError error } testCases := []testCase{ { name: "nominal", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}), + query: ledgercontroller.ColumnPaginatedQuery[any]{}, expected: []ledger.Transaction{tx5, tx4, tx3, tx2, tx1}, }, { name: "address filter", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("account", "bob")), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("account", "bob"), + }, + }, expected: []ledger.Transaction{tx2}, }, { name: "address filter using segments matching two addresses by individual segments", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("account", "users:amazon")), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("account", "users:amazon"), + }, + }, expected: []ledger.Transaction{}, }, { name: "address filter using segment", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("account", "users:")), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("account", "users:"), + }, + }, expected: []ledger.Transaction{tx5, tx4, tx3}, }, { name: "filter using metadata", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("metadata[category]", "2")), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("metadata[category]", "2"), + }, + }, expected: []ledger.Transaction{tx2}, }, { name: "using point in time", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ PIT: pointer.For(now.Add(-time.Hour)), }, - }), + }, expected: []ledger.Transaction{tx3BeforeRevert, tx2, tx1}, }, { name: "filter using invalid key", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("invalid", "2")), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("invalid", "2"), + }, + }, expectError: ledgercontroller.ErrInvalidQuery{}, }, { name: "reverted transactions", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("reverted", true)), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("reverted", true), + }, + }, expected: []ledger.Transaction{tx3}, }, { name: "filter using exists metadata", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Exists("metadata", "category")), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Exists("metadata", "category"), + }, + }, expected: []ledger.Transaction{tx3, tx2, tx1}, }, { name: "filter using metadata and pit", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: pointer.For(tx3.Timestamp), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("metadata[category]", "2"), + PIT: pointer.For(tx3.Timestamp), }, - }). - WithQueryBuilder(query.Match("metadata[category]", "2")), + }, expected: []ledger.Transaction{tx2}, }, { name: "filter using not exists metadata", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Not(query.Exists("metadata", "category"))), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Not(query.Exists("metadata", "category")), + }, + }, expected: []ledger.Transaction{tx5, tx4}, }, { name: "filter using timestamp", - query: ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.PITFilterWithVolumes{}). - WithQueryBuilder(query.Match("timestamp", tx5.Timestamp.Format(time.RFC3339Nano))), + query: ledgercontroller.ColumnPaginatedQuery[any]{ + Options: ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("timestamp", tx5.Timestamp.Format(time.RFC3339Nano)), + }, + }, expected: []ledger.Transaction{tx5, tx4}, }, } @@ -807,10 +852,11 @@ func TestTransactionsList(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() - tc.query.Options.ExpandVolumes = true - tc.query.Options.ExpandEffectiveVolumes = true + store.DumpTables(ctx, "transactions") - cursor, err := store.ListTransactions(ctx, ledgercontroller.NewListTransactionsQuery(tc.query)) + tc.query.Options.Expand = []string{"volumes", "effectiveVolumes"} + + cursor, err := store.Transactions().Paginate(ctx, tc.query) if tc.expectError != nil { require.True(t, errors.Is(err, tc.expectError)) } else { @@ -818,11 +864,11 @@ func TestTransactionsList(t *testing.T) { require.Len(t, cursor.Data, len(tc.expected)) RequireEqual(t, tc.expected, cursor.Data) - count, err := store.CountTransactions(ctx, ledgercontroller.NewListTransactionsQuery(tc.query)) + count, err := store.Transactions().Count(ctx, tc.query.Options) require.NoError(t, err) require.EqualValues(t, len(tc.expected), count) } }) } -} +} \ No newline at end of file diff --git a/internal/storage/ledger/utils.go b/internal/storage/ledger/utils.go index 11b64f438..a7721de7b 100644 --- a/internal/storage/ledger/utils.go +++ b/internal/storage/ledger/utils.go @@ -8,30 +8,20 @@ import ( func isSegmentedAddress(address string) bool { src := strings.Split(address, ":") - needSegmentCheck := false for _, segment := range src { - needSegmentCheck = segment == "" - if needSegmentCheck { - break + if segment == "" { + return true } } - return needSegmentCheck + return false } func filterAccountAddress(address, key string) string { parts := make([]string, 0) - src := strings.Split(address, ":") - needSegmentCheck := false - for _, segment := range src { - needSegmentCheck = segment == "" - if needSegmentCheck { - break - } - } - - if needSegmentCheck { + if isPartialAddress(address) { + src := strings.Split(address, ":") parts = append(parts, fmt.Sprintf("jsonb_array_length(%s_array) = %d", key, len(src))) for i, segment := range src { @@ -46,3 +36,7 @@ func filterAccountAddress(address, key string) string { return strings.Join(parts, " and ") } + +func isPartialAddress(address any) bool { + return isSegmentedAddress(address.(string)) +} diff --git a/internal/storage/ledger/volumes.go b/internal/storage/ledger/volumes.go index 48f207e21..f11932c33 100644 --- a/internal/storage/ledger/volumes.go +++ b/internal/storage/ledger/volumes.go @@ -2,26 +2,18 @@ package ledger import ( "context" - "fmt" "github.com/formancehq/go-libs/v2/collectionutils" "github.com/formancehq/go-libs/v2/platform/postgres" - "github.com/formancehq/ledger/internal/tracing" - "github.com/formancehq/ledger/pkg/features" - - "github.com/formancehq/go-libs/v2/bun/bunpaginate" - lquery "github.com/formancehq/go-libs/v2/query" - "github.com/formancehq/go-libs/v2/time" ledger "github.com/formancehq/ledger/internal" - ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" - "github.com/uptrace/bun" + "github.com/formancehq/ledger/internal/tracing" ) -func (s *Store) UpdateVolumes(ctx context.Context, accountVolumes ...ledger.AccountsVolumes) (ledger.PostCommitVolumes, error) { +func (store *Store) UpdateVolumes(ctx context.Context, accountVolumes ...ledger.AccountsVolumes) (ledger.PostCommitVolumes, error) { return tracing.TraceWithMetric( ctx, "UpdateBalances", - s.tracer, - s.updateBalancesHistogram, + store.tracer, + store.updateBalancesHistogram, func(ctx context.Context) (ledger.PostCommitVolumes, error) { type AccountsVolumesWithLedger struct { @@ -32,13 +24,13 @@ func (s *Store) UpdateVolumes(ctx context.Context, accountVolumes ...ledger.Acco accountsVolumesWithLedger := collectionutils.Map(accountVolumes, func(from ledger.AccountsVolumes) AccountsVolumesWithLedger { return AccountsVolumesWithLedger{ AccountsVolumes: from, - Ledger: s.ledger.Name, + Ledger: store.ledger.Name, } }) - _, err := s.db.NewInsert(). + _, err := store.db.NewInsert(). Model(&accountsVolumesWithLedger). - ModelTableExpr(s.GetPrefixedRelationName("accounts_volumes")). + ModelTableExpr(store.GetPrefixedRelationName("accounts_volumes")). On("conflict (ledger, accounts_address, asset) do update"). Set("input = accounts_volumes.input + excluded.input"). Set("output = accounts_volumes.output + excluded.output"). @@ -63,200 +55,3 @@ func (s *Store) UpdateVolumes(ctx context.Context, accountVolumes ...ledger.Acco }, ) } - -func (s *Store) selectVolumes(oot, pit *time.Time, useInsertionDate bool, groupLevel int, q lquery.Builder) (*bun.SelectQuery, error) { - ret := s.db.NewSelect() - - var ( - useMetadata bool - needSegmentAddress bool - ) - if q != nil { - err := q.Walk(func(operator, key string, value any) error { - switch { - case key == "account" || key == "address": - if err := s.validateAddressFilter(operator, value); err != nil { - return err - } - if !needSegmentAddress { - needSegmentAddress = isSegmentedAddress(value.(string)) // Safe cast - } - case metadataRegex.Match([]byte(key)): - if operator != "$match" { - return ledgercontroller.NewErrInvalidQuery("'metadata' column can only be used with $match") - } - useMetadata = true - case key == "metadata": - if operator != "$exists" { - return ledgercontroller.NewErrInvalidQuery("'metadata' key filter can only be used with $exists") - } - useMetadata = true - case balanceRegex.Match([]byte(key)): - default: - return ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) - } - return nil - }) - if err != nil { - return nil, err - } - } - - var selectVolumes *bun.SelectQuery - - if (pit == nil || pit.IsZero()) && (oot == nil || oot.IsZero()) { - selectVolumes = s.db.NewSelect(). - DistinctOn("accounts_address, asset"). - ColumnExpr("accounts_address as address"). - Column("asset", "input", "output"). - ColumnExpr("input - output as balance"). - ModelTableExpr(s.GetPrefixedRelationName("accounts_volumes")). - Where("ledger = ?", s.ledger.Name). - Order("accounts_address", "asset") - } else { - if !s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { - return nil, ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory) - } - - dateFilterColumn := "effective_date" - if useInsertionDate { - dateFilterColumn = "insertion_date" - } - - selectVolumes = s.db.NewSelect(). - ColumnExpr("accounts_address as address"). - Column("asset"). - ColumnExpr("sum(case when not is_source then amount else 0 end) as input"). - ColumnExpr("sum(case when is_source then amount else 0 end) as output"). - ColumnExpr("sum(case when not is_source then amount else -amount end) as balance"). - ModelTableExpr(s.GetPrefixedRelationName("moves")). - Where("ledger = ?", s.ledger.Name). - GroupExpr("accounts_address, asset"). - Order("accounts_address", "asset") - - if pit != nil && !pit.IsZero() { - selectVolumes = selectVolumes.Where(dateFilterColumn+" <= ?", pit) - } - - if oot != nil && !oot.IsZero() { - selectVolumes = selectVolumes.Where(dateFilterColumn+" >= ?", oot) - } - } - - ret = ret. - ModelTableExpr("(?) volumes", selectVolumes). - Column("address", "asset", "input", "output", "balance") - - if needSegmentAddress { - selectAccount := s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("accounts")). - Where("ledger = ? and address = volumes.address", s.ledger.Name). - Column("address_array") - if useMetadata && (pit == nil || pit.IsZero()) { - selectAccount = selectAccount.Column("metadata") - } - - ret = ret. - Join("join lateral (?) accounts on true", selectAccount). - Column("accounts.address_array") - if useMetadata && (pit == nil || pit.IsZero()) { - ret = ret.Column("accounts.metadata") - } - } - - if useMetadata { - switch { - case needSegmentAddress && (pit == nil || pit.IsZero()): - // nothing to do, already handled earlier - case !needSegmentAddress && (pit == nil || pit.IsZero()): - selectAccount := s.db.NewSelect(). - ModelTableExpr(s.GetPrefixedRelationName("accounts")). - Where("ledger = ? and address = volumes.address", s.ledger.Name). - Column("metadata") - - ret = ret. - Join("join lateral (?) accounts on true", selectAccount). - Column("accounts.metadata") - case pit != nil && !pit.IsZero(): - selectAccountMetadata := s.db.NewSelect(). - Column("metadata"). - ModelTableExpr(s.GetPrefixedRelationName("accounts_metadata")). - Where("ledger = ? and accounts_address = volumes.address and date <= ?", s.ledger.Name, pit) - - ret = ret. - Join("join lateral (?) accounts_metadata on true", selectAccountMetadata). - Column("accounts_metadata.metadata") - } - } - - if q != nil { - where, args, err := q.Build(lquery.ContextFn(func(key, operator string, value any) (string, []any, error) { - - switch { - case key == "account" || key == "address": - return filterAccountAddress(value.(string), "address"), nil, nil - case metadataRegex.Match([]byte(key)): - match := metadataRegex.FindAllStringSubmatch(key, 3) - return "metadata @> ?", []any{map[string]any{ - match[0][1]: value, - }}, nil - case key == "metadata": - return "metadata -> ? is not null", []any{value}, nil - case balanceRegex.Match([]byte(key)): - match := balanceRegex.FindAllStringSubmatch(key, 2) - return `balance ` + convertOperatorToSQL(operator) + ` ? and asset = ?`, []any{value, match[0][1]}, nil - default: - return "", nil, ledgercontroller.NewErrInvalidQuery("unknown key '%s' when building query", key) - } - })) - if err != nil { - return nil, err - } - ret = ret.Where(where, args...) - } - - globalQuery := s.db.NewSelect() - globalQuery = globalQuery. - With("query", ret). - ModelTableExpr("query") - - if groupLevel > 0 { - globalQuery = globalQuery. - ColumnExpr(fmt.Sprintf(`(array_to_string((string_to_array(address, ':'))[1:LEAST(array_length(string_to_array(address, ':'),1),%d)],':')) as account`, groupLevel)). - ColumnExpr("asset"). - ColumnExpr("sum(input) as input"). - ColumnExpr("sum(output) as output"). - ColumnExpr("sum(balance) as balance"). - GroupExpr("account, asset") - } else { - globalQuery = globalQuery.ColumnExpr("address as account, asset, input, output, balance") - } - - return globalQuery, nil -} - -func (s *Store) GetVolumesWithBalances(ctx context.Context, q ledgercontroller.GetVolumesWithBalancesQuery) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { - return tracing.TraceWithMetric( - ctx, - "GetVolumesWithBalances", - s.tracer, - s.getVolumesWithBalancesHistogram, - func(ctx context.Context) (*bunpaginate.Cursor[ledger.VolumesWithBalanceByAssetByAccount], error) { - selectVolumes, err := s.selectVolumes( - q.Options.Options.OOT, - q.Options.Options.PIT, - q.Options.Options.UseInsertionDate, - q.Options.Options.GroupLvl, - q.Options.QueryBuilder, - ) - if err != nil { - return nil, err - } - return bunpaginate.UsingOffset[ledgercontroller.PaginatedQueryOptions[ledgercontroller.FiltersForVolumes], ledger.VolumesWithBalanceByAssetByAccount]( - ctx, - selectVolumes, - bunpaginate.OffsetPaginatedQuery[ledgercontroller.PaginatedQueryOptions[ledgercontroller.FiltersForVolumes]](q), - ) - }, - ) -} diff --git a/internal/storage/ledger/volumes_test.go b/internal/storage/ledger/volumes_test.go index 2da872f8b..27b0e5b82 100644 --- a/internal/storage/ledger/volumes_test.go +++ b/internal/storage/ledger/volumes_test.go @@ -105,27 +105,34 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{UseInsertionDate: true}))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + UseInsertionDate: true, + }, + }, + }) require.NoError(t, err) - require.Len(t, volumes.Data, 4) }) t.Run("Get all volumes with balance for effective date", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions(ledgercontroller.FiltersForVolumes{UseInsertionDate: false}))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{}) require.NoError(t, err) - require.Len(t, volumes.Data, 4) }) t.Run("Get all volumes with balance for insertion date with previous pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &previousPIT, OOT: nil}, - UseInsertionDate: true, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + UseInsertionDate: true, + }, + PIT: &previousPIT, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -142,35 +149,42 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with futur pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: nil}, - UseInsertionDate: true, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + UseInsertionDate: true, + }, + PIT: &futurPIT, + }, + }) require.NoError(t, err) - require.Len(t, volumes.Data, 4) }) t.Run("Get all volumes with balance for insertion date with previous oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &previousOOT}, - UseInsertionDate: true, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + UseInsertionDate: true, + }, + OOT: &previousOOT, + }, + }) require.NoError(t, err) - require.Len(t, volumes.Data, 4) }) t.Run("Get all volumes with balance for insertion date with future oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &futurOOT}, - UseInsertionDate: true, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + UseInsertionDate: true, + }, + OOT: &futurOOT, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -187,11 +201,11 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with previous pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &previousPIT, OOT: nil}, - UseInsertionDate: false, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &previousPIT, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -208,35 +222,33 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for effective date with futur pit", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: nil}, - UseInsertionDate: false, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &futurPIT, + }, + }) require.NoError(t, err) - require.Len(t, volumes.Data, 4) }) t.Run("Get all volumes with balance for effective date with previous oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &previousOOT}, - UseInsertionDate: false, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + OOT: &previousOOT, + }, + }) require.NoError(t, err) - require.Len(t, volumes.Data, 4) }) t.Run("Get all volumes with balance for effective date with futur oot", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: nil, OOT: &futurOOT}, - UseInsertionDate: false, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + OOT: &futurOOT, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -253,11 +265,15 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with future PIT and now OOT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: &now}, - UseInsertionDate: true, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + UseInsertionDate: true, + }, + PIT: &futurPIT, + OOT: &now, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -270,16 +286,19 @@ func TestVolumesList(t *testing.T) { Balance: big.NewInt(-50), }, }, volumes.Data[0]) - }) t.Run("Get all volumes with balance for insertion date with previous OOT and now PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &now, OOT: &previousOOT}, - UseInsertionDate: true, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + UseInsertionDate: true, + }, + PIT: &now, + OOT: &previousOOT, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -292,16 +311,16 @@ func TestVolumesList(t *testing.T) { Balance: big.NewInt(50), }, }, volumes.Data[0]) - }) t.Run("Get all volumes with balance for effective date with future PIT and now OOT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &futurPIT, OOT: &now}, - UseInsertionDate: false, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &futurPIT, + OOT: &now, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 3) @@ -318,11 +337,12 @@ func TestVolumesList(t *testing.T) { t.Run("Get all volumes with balance for insertion date with previous OOT and now PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery(ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &now, OOT: &previousOOT}, - UseInsertionDate: false, - }))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &now, + OOT: &previousOOT, + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 4) @@ -335,21 +355,20 @@ func TestVolumesList(t *testing.T) { Balance: big.NewInt(-50), }, }, volumes.Data[0]) - }) t.Run("Get account1 volume and Balance for insertion date with previous OOT and now PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{PIT: &now, OOT: &previousOOT}, - UseInsertionDate: false, - }).WithQueryBuilder(query.Match("account", "account:1"))), + volumes, err := store.Volumes().Paginate(ctx, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + PIT: &now, + OOT: &previousOOT, + Builder: query.Match("account", "account:1"), + }, + }, ) - require.NoError(t, err) require.Len(t, volumes.Data, 1) require.Equal(t, ledger.VolumesWithBalanceByAssetByAccount{ @@ -367,26 +386,27 @@ func TestVolumesList(t *testing.T) { t.Run("Using Metadata regex", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{}).WithQueryBuilder(query.Match("metadata[foo]", "bar"))), + volumes, err := store.Volumes().Paginate(ctx, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Builder: query.Match("metadata[foo]", "bar"), + }, + }, ) - require.NoError(t, err) require.Len(t, volumes.Data, 1) - }) t.Run("Using exists metadata filter 1", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{}).WithQueryBuilder(query.Exists("metadata", "category"))), + volumes, err := store.Volumes().Paginate(ctx, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Builder: query.Exists("metadata", "category"), + }, + }, ) - require.NoError(t, err) require.Len(t, volumes.Data, 2) }) @@ -394,12 +414,13 @@ func TestVolumesList(t *testing.T) { t.Run("Using exists metadata filter 2", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{}).WithQueryBuilder(query.Exists("metadata", "foo"))), + volumes, err := store.Volumes().Paginate(ctx, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Builder: query.Exists("metadata", "foo"), + }, + }, ) - require.NoError(t, err) require.Len(t, volumes.Data, 1) }) @@ -465,52 +486,59 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 0", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, - GroupLvl: 0, - }).WithQueryBuilder(query.Match("account", "account::")))) - + }, + Builder: query.Match("account", "account::"), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 7) }) t.Run("Aggregation Volumes with balance for GroupLvl 1", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, GroupLvl: 1, - }).WithQueryBuilder(query.Match("account", "account::")))) - + }, + Builder: query.Match("account", "account::"), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 2) }) t.Run("Aggregation Volumes with balance for GroupLvl 2", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, GroupLvl: 2, - }).WithQueryBuilder(query.Match("account", "account::")))) - + }, + Builder: query.Match("account", "account::"), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 4) }) t.Run("Aggregation Volumes with balance for GroupLvl 3", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ UseInsertionDate: true, GroupLvl: 3, - }).WithQueryBuilder(query.Match("account", "account::")))) - + }, + Builder: query.Match("account", "account::"), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 7) }) @@ -518,16 +546,16 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && PIT && OOT && effectiveDate", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &pit, - OOT: &oot, - }, + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, - }).WithQueryBuilder(query.Match("account", "account::")))) - + }, + PIT: &pit, + OOT: &oot, + Builder: query.Match("account", "account::"), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 2) require.Equal(t, volumes.Data[0], ledger.VolumesWithBalanceByAssetByAccount{ @@ -552,18 +580,17 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && PIT && OOT && effectiveDate && Balance Filter 1", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{ - PIT: &pit, - OOT: &oot, - }, - UseInsertionDate: false, - GroupLvl: 1, - }).WithQueryBuilder( - query.And(query.Match("account", "account::"), query.Gte("balance[EUR]", 50))))) + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ + GroupLvl: 1, + }, + PIT: &pit, + OOT: &oot, + Builder: query.And(query.Match("account", "account::"), query.Gte("balance[EUR]", 50)), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 1) require.Equal(t, volumes.Data[0], ledger.VolumesWithBalanceByAssetByAccount{ @@ -579,17 +606,17 @@ func TestVolumesAggregate(t *testing.T) { t.Run("Aggregation Volumes with balance for GroupLvl 1 && Balance Filter 2", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ - PITFilter: ledgercontroller.PITFilter{}, - UseInsertionDate: true, + volumes, err := store.Volumes().Paginate(ctx, ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 2, - }).WithQueryBuilder( - query.Or( + UseInsertionDate: true, + }, + Builder: query.Or( query.Match("account", "account:1:"), - query.Lte("balance[USD]", 0))))) - + query.Lte("balance[USD]", 0)), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 3) require.Equal(t, volumes.Data[0], ledger.VolumesWithBalanceByAssetByAccount{ @@ -623,16 +650,18 @@ func TestVolumesAggregate(t *testing.T) { t.Run("filter using account matching, metadata, and group", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + volumes, err := store.Volumes().Paginate(ctx, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, - }).WithQueryBuilder(query.And( - query.Match("account", "account::"), - query.Match("metadata[foo]", "bar"), - ))), - ) + }, + Builder: query.And( + query.Match("account", "account::"), + query.Match("metadata[foo]", "bar"), + ), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 1) @@ -641,19 +670,19 @@ func TestVolumesAggregate(t *testing.T) { t.Run("filter using account matching, metadata, and group and PIT", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + volumes, err := store.Volumes().Paginate(ctx, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, - PITFilter: ledgercontroller.PITFilter{ - PIT: pointer.For(now.Add(time.Minute)), - }, - }).WithQueryBuilder(query.And( - query.Match("account", "account::"), - query.Match("metadata[foo]", "bar"), - ))), - ) + }, + PIT: pointer.For(now.Add(time.Minute)), + Builder: query.And( + query.Match("account", "account::"), + query.Match("metadata[foo]", "bar"), + ), + }, + }) require.NoError(t, err) require.Len(t, volumes.Data, 1) @@ -662,16 +691,16 @@ func TestVolumesAggregate(t *testing.T) { t.Run("filter using metadata matching only", func(t *testing.T) { t.Parallel() - volumes, err := store.GetVolumesWithBalances(ctx, - ledgercontroller.NewGetVolumesWithBalancesQuery( - ledgercontroller.NewPaginatedQueryOptions( - ledgercontroller.FiltersForVolumes{ + volumes, err := store.Volumes().Paginate(ctx, + ledgercontroller.OffsetPaginatedQuery[ledgercontroller.GetVolumesOptions]{ + Options: ledgercontroller.ResourceQuery[ledgercontroller.GetVolumesOptions]{ + Opts: ledgercontroller.GetVolumesOptions{ GroupLvl: 1, - }).WithQueryBuilder(query.And( - query.Match("metadata[foo]", "bar"), - ))), + }, + Builder: query.Match("metadata[foo]", "bar"), + }, + }, ) - require.NoError(t, err) require.Len(t, volumes.Data, 1) }) diff --git a/internal/volumes.go b/internal/volumes.go index 3abcca42b..1c96bece4 100644 --- a/internal/volumes.go +++ b/internal/volumes.go @@ -153,3 +153,7 @@ func (a PostCommitVolumes) Merge(volumes PostCommitVolumes) PostCommitVolumes { return a } + +type AggregatedVolumes struct { + Aggregated VolumesByAssets `bun:"aggregated,type:jsonb"` +} diff --git a/test/e2e/api_transactions_list_test.go b/test/e2e/api_transactions_list_test.go index a4ff42884..d162b7367 100644 --- a/test/e2e/api_transactions_list_test.go +++ b/test/e2e/api_transactions_list_test.go @@ -6,7 +6,10 @@ import ( "fmt" "github.com/formancehq/go-libs/v2/bun/bunpaginate" "github.com/formancehq/go-libs/v2/logging" + "github.com/formancehq/go-libs/v2/query" . "github.com/formancehq/go-libs/v2/testing/api" + libtime "github.com/formancehq/go-libs/v2/time" + ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" "github.com/formancehq/ledger/pkg/client/models/components" "github.com/formancehq/ledger/pkg/client/models/operations" . "github.com/formancehq/ledger/pkg/testserver" @@ -46,7 +49,7 @@ var _ = Context("Ledger transactions list API tests", func() { ) When(fmt.Sprintf("creating %d transactions", txCount), func() { var ( - timestamp = time.Now().Round(time.Second).UTC() + timestamp = time.Now() transactions []components.V2Transaction ) JustBeforeEach(func() { @@ -73,6 +76,12 @@ var _ = Context("Ledger transactions list API tests", func() { Source: "world", Destination: fmt.Sprintf("account:%d", i), }, + { + Amount: big.NewInt(100), + Asset: "EUR", + Source: "world", + Destination: fmt.Sprintf("account:%d", i), + }, }, Timestamp: pointer.For(txTimestamp), }, @@ -193,7 +202,6 @@ var _ = Context("Ledger transactions list API tests", func() { operations.V2ListTransactionsRequest{ Cursor: rsp.Previous, Ledger: "default", - Expand: pointer.For("volumes,effectiveVolumes"), }, ) Expect(err).ToNot(HaveOccurred()) @@ -232,21 +240,12 @@ var _ = Context("Ledger transactions list API tests", func() { }) It("Should be ok", func() { Expect(response.Next).NotTo(BeNil()) - cursor := &bunpaginate.ColumnPaginatedQuery[map[string]any]{} + cursor := &ledgercontroller.ColumnPaginatedQuery[any]{} Expect(bunpaginate.UnmarshalCursor(*response.Next, cursor)).To(BeNil()) - Expect(cursor.Options).To(Equal(map[string]any{ - "qb": map[string]any{ - "$match": map[string]any{ - "source": "world", - }, - }, - "pageSize": float64(10), - "options": map[string]any{ - "pit": now.Format(time.RFC3339), - "oot": nil, - "volumes": false, - "effectiveVolumes": false, - }, + Expect(cursor.PageSize).To(Equal(uint64(10))) + Expect(cursor.Options).To(Equal(ledgercontroller.ResourceQuery[any]{ + Builder: query.Match("source", "world"), + PIT: pointer.For(libtime.New(now)), })) }) }) @@ -284,30 +283,15 @@ var _ = Context("Ledger transactions list API tests", func() { }) It("Should be ok", func() { Expect(response.Next).NotTo(BeNil()) - cursor := &bunpaginate.ColumnPaginatedQuery[map[string]any]{} + cursor := &ledgercontroller.ColumnPaginatedQuery[any]{} Expect(bunpaginate.UnmarshalCursor(*response.Next, cursor)).To(BeNil()) - Expect(cursor.Options).To(Equal(map[string]any{ - "qb": map[string]any{ - "$and": []any{ - map[string]any{ - "$match": map[string]any{ - "source": "world", - }, - }, - map[string]any{ - "$match": map[string]any{ - "destination": "account:", - }, - }, - }, - }, - "pageSize": float64(10), - "options": map[string]any{ - "pit": now.Format(time.RFC3339), - "oot": nil, - "volumes": false, - "effectiveVolumes": false, - }, + Expect(cursor.PageSize).To(Equal(uint64(10))) + Expect(cursor.Options).To(Equal(ledgercontroller.ResourceQuery[any]{ + Builder: query.And( + query.Match("source", "world"), + query.Match("destination", "account:"), + ), + PIT: pointer.For(libtime.New(now)), })) }) }) diff --git a/tools/generator/go.sum b/tools/generator/go.sum index 5241a1b9e..53074ed2c 100644 --- a/tools/generator/go.sum +++ b/tools/generator/go.sum @@ -280,6 +280,8 @@ github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=