From 18d7e1c4f7ad99941ae0eec6db1c15b3a71e5591 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Tue, 19 Nov 2024 14:51:07 +0100 Subject: [PATCH] feat: add generator set --- docs/api/README.md | 46 +++++ internal/README.md | 120 +----------- internal/api/common/mocks.go | 2 +- internal/api/v1/mocks.go | 2 +- internal/api/v2/common.go | 2 +- internal/api/v2/errors.go | 16 +- internal/api/v2/mocks.go | 2 +- .../controller/ledger/controller_default.go | 3 +- internal/controller/system/controller.go | 3 +- internal/controller/system/errors.go | 4 +- internal/errors.go | 2 +- internal/ledger.go | 128 +------------ internal/ledger_test.go | 5 +- internal/storage/bucket/bucket.go | 81 ++++---- internal/storage/ledger/accounts.go | 11 +- internal/storage/ledger/balances.go | 15 +- internal/storage/ledger/logs.go | 3 +- internal/storage/ledger/moves.go | 5 +- internal/storage/ledger/store.go | 5 +- internal/storage/ledger/transactions.go | 13 +- internal/storage/ledger/volumes.go | 5 +- internal/storage/module.go | 8 +- openapi.yaml | 24 +++ openapi/v2.yaml | 24 +++ pkg/client/.speakeasy/gen.lock | 8 +- pkg/client/.speakeasy/gen.yaml | 2 +- pkg/client/README.md | 1 + .../models/operations/getmetricsresponse.md | 9 + pkg/client/docs/sdks/ledger/README.md | 52 +++++ pkg/client/formance.go | 4 +- pkg/client/ledger.go | 179 ++++++++++++++++++ pkg/client/models/operations/getmetrics.go | 27 +++ pkg/features/features.go | 119 ++++++++++++ pkg/generate/generator.go | 10 +- pkg/generate/set.go | 73 +++++++ pkg/testserver/server.go | 2 +- test/e2e/api_ledgers_create_test.go | 12 +- test/e2e/api_ledgers_import_test.go | 6 +- test/performance/benchmark_test.go | 13 +- test/performance/features_test.go | 14 +- test/rolling-upgrades/main_test.go | 6 +- test/stress/stress_test.go | 4 +- tools/generator/cmd/root.go | 45 +---- 43 files changed, 712 insertions(+), 403 deletions(-) create mode 100644 pkg/client/docs/models/operations/getmetricsresponse.md create mode 100644 pkg/client/models/operations/getmetrics.go create mode 100644 pkg/features/features.go create mode 100644 pkg/generate/set.go diff --git a/docs/api/README.md b/docs/api/README.md index 616d7e693..613b21baa 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -60,6 +60,52 @@ To perform this operation, you must be authenticated by means of one of the foll Authorization ( Scopes: ledger:read ) +## Read in memory metrics + + + +> Code samples + +```http +GET http://localhost:8080/_/metrics HTTP/1.1 +Host: localhost:8080 +Accept: application/json + +``` + +`GET /_/metrics` + +> Example responses + +> 200 Response + +```json +{ + "property1": null, + "property2": null +} +``` + +

Responses

+ +|Status|Meaning|Description|Schema| +|---|---|---|---| +|200|[OK](https://tools.ietf.org/html/rfc7231#section-6.3.1)|OK|Inline| +|default|Default|Error|[V2ErrorResponse](#schemav2errorresponse)| + +

Response Schema

+ +Status Code **200** + +|Name|Type|Required|Restrictions|Description| +|---|---|---|---|---| +|ยป **additionalProperties**|any|false|none|none| + + +

ledger.v2

## List ledgers diff --git a/internal/README.md b/internal/README.md index d71e81cf6..6a8490675 100644 --- a/internal/README.md +++ b/internal/README.md @@ -32,11 +32,6 @@ import "github.com/formancehq/ledger/internal" - [type ErrInvalidLedgerName](<#ErrInvalidLedgerName>) - [func \(e ErrInvalidLedgerName\) Error\(\) string](<#ErrInvalidLedgerName.Error>) - [func \(e ErrInvalidLedgerName\) Is\(err error\) bool](<#ErrInvalidLedgerName.Is>) -- [type FeatureSet](<#FeatureSet>) - - [func \(f FeatureSet\) Match\(features FeatureSet\) bool](<#FeatureSet.Match>) - - [func \(f FeatureSet\) SortedKeys\(\) \[\]string](<#FeatureSet.SortedKeys>) - - [func \(f FeatureSet\) String\(\) string](<#FeatureSet.String>) - - [func \(f FeatureSet\) With\(feature, value string\) FeatureSet](<#FeatureSet.With>) - [type Ledger](<#Ledger>) - [func MustNewWithDefault\(name string\) Ledger](<#MustNewWithDefault>) - [func New\(name string, configuration Configuration\) \(\*Ledger, error\)](<#New>) @@ -116,40 +111,20 @@ import "github.com/formancehq/ledger/internal" ## Constants - + ```go const ( - // FeatureMovesHistory is used to define if the ledger has to save funds movements history. - // Value is either ON or OFF - FeatureMovesHistory = "MOVES_HISTORY" - // FeatureMovesHistoryPostCommitEffectiveVolumes is used to define if the pvce property of funds movements history - // has to be updated with back dated transaction. - // Value is either SYNC or DISABLED. - // todo: depends on FeatureMovesHistory (dependency should be checked) - FeatureMovesHistoryPostCommitEffectiveVolumes = "MOVES_HISTORY_POST_COMMIT_EFFECTIVE_VOLUMES" - // FeatureHashLogs is used to defined it the logs has to be hashed. - FeatureHashLogs = "HASH_LOGS" - // FeatureAccountMetadataHistory is used to defined it the account metadata must be historized. - FeatureAccountMetadataHistory = "ACCOUNT_METADATA_HISTORY" - // FeatureTransactionMetadataHistory is used to defined it the transaction metadata must be historized. - FeatureTransactionMetadataHistory = "TRANSACTION_METADATA_HISTORY" - // FeatureIndexAddressSegments is used to defined it we want to index segments of accounts address. - // Without this feature, the ledger will not allow filtering on partial account address. - FeatureIndexAddressSegments = "INDEX_ADDRESS_SEGMENTS" - // FeatureIndexTransactionAccounts is used to defined it we want to index accounts used in a transaction. - FeatureIndexTransactionAccounts = "INDEX_TRANSACTION_ACCOUNTS" - - DefaultBucket = "_default" + MetaTargetTypeAccount = "ACCOUNT" + MetaTargetTypeTransaction = "TRANSACTION" ) ``` - + ```go const ( - MetaTargetTypeAccount = "ACCOUNT" - MetaTargetTypeTransaction = "TRANSACTION" + DefaultBucket = "_default" ) ``` @@ -163,40 +138,6 @@ const ( ## Variables - - -```go -var ( - DefaultFeatures = FeatureSet{ - FeatureMovesHistory: "ON", - FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", - FeatureHashLogs: "SYNC", - FeatureAccountMetadataHistory: "SYNC", - FeatureTransactionMetadataHistory: "SYNC", - FeatureIndexAddressSegments: "ON", - FeatureIndexTransactionAccounts: "ON", - } - MinimalFeatureSet = FeatureSet{ - FeatureMovesHistory: "OFF", - FeatureMovesHistoryPostCommitEffectiveVolumes: "DISABLED", - FeatureHashLogs: "DISABLED", - FeatureAccountMetadataHistory: "DISABLED", - FeatureTransactionMetadataHistory: "DISABLED", - FeatureIndexAddressSegments: "OFF", - FeatureIndexTransactionAccounts: "OFF", - } - FeatureConfigurations = map[string][]string{ - FeatureMovesHistory: {"ON", "OFF"}, - FeatureMovesHistoryPostCommitEffectiveVolumes: {"SYNC", "DISABLED"}, - FeatureHashLogs: {"SYNC", "DISABLED"}, - FeatureAccountMetadataHistory: {"SYNC", "DISABLED"}, - FeatureTransactionMetadataHistory: {"SYNC", "DISABLED"}, - FeatureIndexAddressSegments: {"ON", "OFF"}, - FeatureIndexTransactionAccounts: {"ON", "OFF"}, - } -) -``` - ```go @@ -281,9 +222,9 @@ type BalancesByAssetsByAccounts map[string]BalancesByAssets ```go type Configuration struct { - Bucket string `json:"bucket" bun:"bucket,type:varchar(255)"` - Metadata metadata.Metadata `json:"metadata" bun:"metadata,type:jsonb"` - Features FeatureSet `json:"features" bun:"features,type:jsonb"` + Bucket string `json:"bucket" bun:"bucket,type:varchar(255)"` + Metadata metadata.Metadata `json:"metadata" bun:"metadata,type:jsonb"` + Features features.FeatureSet `json:"features" bun:"features,type:jsonb"` } ``` @@ -433,51 +374,6 @@ func (e ErrInvalidLedgerName) Is(err error) bool - -## type FeatureSet - - - -```go -type FeatureSet map[string]string -``` - - -### func \(FeatureSet\) Match - -```go -func (f FeatureSet) Match(features FeatureSet) bool -``` - - - - -### func \(FeatureSet\) SortedKeys - -```go -func (f FeatureSet) SortedKeys() []string -``` - - - - -### func \(FeatureSet\) String - -```go -func (f FeatureSet) String() string -``` - - - - -### func \(FeatureSet\) With - -```go -func (f FeatureSet) With(feature, value string) FeatureSet -``` - - - ## type Ledger diff --git a/internal/api/common/mocks.go b/internal/api/common/mocks.go index ffc0edce4..349e3fa56 100644 --- a/internal/api/common/mocks.go +++ b/internal/api/common/mocks.go @@ -1,3 +1,3 @@ //go:generate mockgen -write_source_comment=false -write_package_comment=false -source ../../controller/system/controller.go -destination mocks_system_controller_test.go -package common --mock_names Controller=SystemController . Controller //go:generate mockgen -write_source_comment=false -write_package_comment=false -source ../../controller/ledger/controller.go -destination mocks_ledger_controller_test.go -package common --mock_names Controller=LedgerController . Controller -package common \ No newline at end of file +package common diff --git a/internal/api/v1/mocks.go b/internal/api/v1/mocks.go index f10db2ce4..f2523b685 100644 --- a/internal/api/v1/mocks.go +++ b/internal/api/v1/mocks.go @@ -1,3 +1,3 @@ //go:generate mockgen -write_source_comment=false -write_package_comment=false -source ../../controller/system/controller.go -destination mocks_system_controller_test.go -package v1 --mock_names Controller=SystemController . Controller //go:generate mockgen -write_source_comment=false -write_package_comment=false -source ../../controller/ledger/controller.go -destination mocks_ledger_controller_test.go -package v1 --mock_names Controller=LedgerController . Controller -package v1 \ No newline at end of file +package v1 diff --git a/internal/api/v2/common.go b/internal/api/v2/common.go index 82fbc486e..b223f1fee 100644 --- a/internal/api/v2/common.go +++ b/internal/api/v2/common.go @@ -176,4 +176,4 @@ func getPaginatedQueryOptionsOfFiltersForVolumes(r *http.Request) (*ledgercontro return pointer.For(ledgercontroller.NewPaginatedQueryOptions(*filtersForVolumes). WithPageSize(pageSize). WithQueryBuilder(qb)), nil -} \ No newline at end of file +} diff --git a/internal/api/v2/errors.go b/internal/api/v2/errors.go index 438a1e9ad..cdcc2b254 100644 --- a/internal/api/v2/errors.go +++ b/internal/api/v2/errors.go @@ -1,14 +1,14 @@ package v2 const ( - ErrConflict = "CONFLICT" - ErrInsufficientFund = "INSUFFICIENT_FUND" - ErrValidation = "VALIDATION" - ErrAlreadyRevert = "ALREADY_REVERT" - ErrNoPostings = "NO_POSTINGS" - ErrCompilationFailed = "COMPILATION_FAILED" - ErrMetadataOverride = "METADATA_OVERRIDE" - ErrBulkSizeExceeded = "BULK_SIZE_EXCEEDED" + ErrConflict = "CONFLICT" + ErrInsufficientFund = "INSUFFICIENT_FUND" + ErrValidation = "VALIDATION" + ErrAlreadyRevert = "ALREADY_REVERT" + ErrNoPostings = "NO_POSTINGS" + ErrCompilationFailed = "COMPILATION_FAILED" + ErrMetadataOverride = "METADATA_OVERRIDE" + ErrBulkSizeExceeded = "BULK_SIZE_EXCEEDED" ErrLedgerAlreadyExists = "LEDGER_ALREADY_EXISTS" ErrInterpreterParse = "INTERPRETER_PARSE" diff --git a/internal/api/v2/mocks.go b/internal/api/v2/mocks.go index c082bd6a6..2cfde480e 100644 --- a/internal/api/v2/mocks.go +++ b/internal/api/v2/mocks.go @@ -1,3 +1,3 @@ //go:generate mockgen -write_source_comment=false -write_package_comment=false -source ../../controller/system/controller.go -destination mocks_system_controller_test.go -package v2 --mock_names Controller=SystemController . Controller //go:generate mockgen -write_source_comment=false -write_package_comment=false -source ../../controller/ledger/controller.go -destination mocks_ledger_controller_test.go -package v2 --mock_names Controller=LedgerController . Controller -package v2 \ No newline at end of file +package v2 diff --git a/internal/controller/ledger/controller_default.go b/internal/controller/ledger/controller_default.go index f8dcc70ef..b55d29e78 100644 --- a/internal/controller/ledger/controller_default.go +++ b/internal/controller/ledger/controller_default.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "fmt" + "github.com/formancehq/ledger/pkg/features" "math/big" "reflect" @@ -210,7 +211,7 @@ func (ctrl *DefaultController) importLog(ctx context.Context, sqlTx TX, log ledg return fmt.Errorf("failed to insert log: %w", err) } - if ctrl.ledger.HasFeature(ledger.FeatureHashLogs, "SYNC") { + if ctrl.ledger.HasFeature(features.FeatureHashLogs, "SYNC") { if !reflect.DeepEqual(log.Hash, logCopy.Hash) { return newErrInvalidHash(log.ID, logCopy.Hash, log.Hash) } diff --git a/internal/controller/system/controller.go b/internal/controller/system/controller.go index 86b6b63d7..44265079f 100644 --- a/internal/controller/system/controller.go +++ b/internal/controller/system/controller.go @@ -2,6 +2,7 @@ package system import ( "context" + "github.com/formancehq/ledger/pkg/features" "reflect" "time" @@ -89,7 +90,7 @@ func (ctrl *DefaultController) CreateLedger(ctx context.Context, name string, co configuration.SetDefaults() if !ctrl.enableFeatures { - if !reflect.DeepEqual(configuration.Features, ledger.DefaultFeatures) { + if !reflect.DeepEqual(configuration.Features, features.DefaultFeatures) { return ErrExperimentalFeaturesDisabled } } diff --git a/internal/controller/system/errors.go b/internal/controller/system/errors.go index d5b5e1da7..18d5eca64 100644 --- a/internal/controller/system/errors.go +++ b/internal/controller/system/errors.go @@ -6,7 +6,7 @@ import ( ) var ( - ErrLedgerAlreadyExists = errors.New("ledger already exists") + ErrLedgerAlreadyExists = errors.New("ledger already exists") ErrExperimentalFeaturesDisabled = errors.New("experimental features are disabled") ) @@ -27,4 +27,4 @@ func newErrInvalidLedgerConfiguration(err error) ErrInvalidLedgerConfiguration { return ErrInvalidLedgerConfiguration{ err: err, } -} \ No newline at end of file +} diff --git a/internal/errors.go b/internal/errors.go index f51e711c4..aa6d9e777 100644 --- a/internal/errors.go +++ b/internal/errors.go @@ -36,4 +36,4 @@ func (e ErrInvalidBucketName) Is(err error) bool { func newErrInvalidBucketName(bucket string, err error) ErrInvalidBucketName { return ErrInvalidBucketName{err: err, bucket: bucket} -} \ No newline at end of file +} diff --git a/internal/ledger.go b/internal/ledger.go index a66f5e996..f50708fd2 100644 --- a/internal/ledger.go +++ b/internal/ledger.go @@ -2,14 +2,12 @@ package ledger import ( "fmt" - . "github.com/formancehq/go-libs/v2/collectionutils" + "github.com/formancehq/go-libs/v2/metadata" "github.com/formancehq/go-libs/v2/time" + "github.com/formancehq/ledger/pkg/features" "github.com/uptrace/bun" "regexp" "slices" - "strings" - - "github.com/formancehq/go-libs/v2/metadata" ) type Ledger struct { @@ -22,7 +20,7 @@ type Ledger struct { } func (l Ledger) HasFeature(feature, value string) bool { - if err := validateFeatureWithValue(feature, value); err != nil { + if err := features.ValidateFeatureWithValue(feature, value); err != nil { panic(err) } @@ -69,58 +67,10 @@ func MustNewWithDefault(name string) Ledger { } const ( - // FeatureMovesHistory is used to define if the ledger has to save funds movements history. - // Value is either ON or OFF - FeatureMovesHistory = "MOVES_HISTORY" - // FeatureMovesHistoryPostCommitEffectiveVolumes is used to define if the pvce property of funds movements history - // has to be updated with back dated transaction. - // Value is either SYNC or DISABLED. - // todo: depends on FeatureMovesHistory (dependency should be checked) - FeatureMovesHistoryPostCommitEffectiveVolumes = "MOVES_HISTORY_POST_COMMIT_EFFECTIVE_VOLUMES" - // FeatureHashLogs is used to defined it the logs has to be hashed. - FeatureHashLogs = "HASH_LOGS" - // FeatureAccountMetadataHistory is used to defined it the account metadata must be historized. - FeatureAccountMetadataHistory = "ACCOUNT_METADATA_HISTORY" - // FeatureTransactionMetadataHistory is used to defined it the transaction metadata must be historized. - FeatureTransactionMetadataHistory = "TRANSACTION_METADATA_HISTORY" - // FeatureIndexAddressSegments is used to defined it we want to index segments of accounts address. - // Without this feature, the ledger will not allow filtering on partial account address. - FeatureIndexAddressSegments = "INDEX_ADDRESS_SEGMENTS" - // FeatureIndexTransactionAccounts is used to defined it we want to index accounts used in a transaction. - FeatureIndexTransactionAccounts = "INDEX_TRANSACTION_ACCOUNTS" - DefaultBucket = "_default" ) var ( - DefaultFeatures = FeatureSet{ - FeatureMovesHistory: "ON", - FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", - FeatureHashLogs: "SYNC", - FeatureAccountMetadataHistory: "SYNC", - FeatureTransactionMetadataHistory: "SYNC", - FeatureIndexAddressSegments: "ON", - FeatureIndexTransactionAccounts: "ON", - } - MinimalFeatureSet = FeatureSet{ - FeatureMovesHistory: "OFF", - FeatureMovesHistoryPostCommitEffectiveVolumes: "DISABLED", - FeatureHashLogs: "DISABLED", - FeatureAccountMetadataHistory: "DISABLED", - FeatureTransactionMetadataHistory: "DISABLED", - FeatureIndexAddressSegments: "OFF", - FeatureIndexTransactionAccounts: "OFF", - } - FeatureConfigurations = map[string][]string{ - FeatureMovesHistory: {"ON", "OFF"}, - FeatureMovesHistoryPostCommitEffectiveVolumes: {"SYNC", "DISABLED"}, - FeatureHashLogs: {"SYNC", "DISABLED"}, - FeatureAccountMetadataHistory: {"SYNC", "DISABLED"}, - FeatureTransactionMetadataHistory: {"SYNC", "DISABLED"}, - FeatureIndexAddressSegments: {"ON", "OFF"}, - FeatureIndexTransactionAccounts: {"ON", "OFF"}, - } - ledgerNameFormat = regexp.MustCompile("^[0-9a-zA-Z_-]{1,63}$") bucketNameFormat = regexp.MustCompile("^[0-9a-zA-Z_-]{1,63}$") @@ -132,70 +82,10 @@ var ( } ) -func validateFeatureWithValue(feature, value string) error { - possibleConfigurations, ok := FeatureConfigurations[feature] - if !ok { - return fmt.Errorf("feature %q not exists", feature) - } - if !slices.Contains(possibleConfigurations, value) { - return fmt.Errorf("configuration %s it not possible for feature %s", value, feature) - } - - return nil -} - -type FeatureSet map[string]string - -func (f FeatureSet) With(feature, value string) FeatureSet { - ret := FeatureSet{} - for k, v := range f { - ret[k] = v - } - ret[feature] = value - - return ret -} - -func (f FeatureSet) SortedKeys() []string { - ret := Keys(f) - slices.Sort(ret) - - return ret -} - -func (f FeatureSet) String() string { - if len(f) == 0 { - return "" - } - - ret := "" - for _, key := range f.SortedKeys() { - ret = ret + "," + shortenFeature(key) + "=" + f[key] - } - - return ret[1:] -} - -func (f FeatureSet) Match(features FeatureSet) bool { - for key, value := range features { - if f[key] != value { - return false - } - } - - return true -} - -func shortenFeature(feature string) string { - return strings.Join(Map(strings.Split(feature, "_"), func(from string) string { - return from[:1] - }), "") -} - type Configuration struct { - Bucket string `json:"bucket" bun:"bucket,type:varchar(255)"` - Metadata metadata.Metadata `json:"metadata" bun:"metadata,type:jsonb"` - Features FeatureSet `json:"features" bun:"features,type:jsonb"` + Bucket string `json:"bucket" bun:"bucket,type:varchar(255)"` + Metadata metadata.Metadata `json:"metadata" bun:"metadata,type:jsonb"` + Features features.FeatureSet `json:"features" bun:"features,type:jsonb"` } func (c *Configuration) SetDefaults() { @@ -206,7 +96,7 @@ func (c *Configuration) SetDefaults() { c.Features = map[string]string{} } - for key, value := range DefaultFeatures { + for key, value := range features.DefaultFeatures { if _, ok := c.Features[key]; !ok { c.Features[key] = value } @@ -215,7 +105,7 @@ func (c *Configuration) SetDefaults() { func (c *Configuration) Validate() error { for feature, value := range c.Features { - if err := validateFeatureWithValue(feature, value); err != nil { + if err := features.ValidateFeatureWithValue(feature, value); err != nil { return err } } @@ -227,6 +117,6 @@ func NewDefaultConfiguration() Configuration { return Configuration{ Bucket: DefaultBucket, Metadata: metadata.Metadata{}, - Features: DefaultFeatures, + Features: features.DefaultFeatures, } } diff --git a/internal/ledger_test.go b/internal/ledger_test.go index 91c6534a8..588e6821d 100644 --- a/internal/ledger_test.go +++ b/internal/ledger_test.go @@ -1,12 +1,13 @@ package ledger import ( + "github.com/formancehq/ledger/pkg/features" "github.com/stretchr/testify/require" "testing" ) func TestFeatures(t *testing.T) { - f := MinimalFeatureSet.With(FeatureMovesHistory, "DISABLED") - require.Equal(t, "DISABLED", f[FeatureMovesHistory]) + f := features.MinimalFeatureSet.With(features.FeatureMovesHistory, "DISABLED") + require.Equal(t, "DISABLED", f[features.FeatureMovesHistory]) require.Equal(t, "AMH=DISABLED,HL=DISABLED,IAS=OFF,ITA=OFF,MH=DISABLED,MHPCEV=DISABLED,TMH=DISABLED", f.String()) } diff --git a/internal/storage/bucket/bucket.go b/internal/storage/bucket/bucket.go index fd6e5422a..2570750e2 100644 --- a/internal/storage/bucket/bucket.go +++ b/internal/storage/bucket/bucket.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/formancehq/go-libs/v2/migrations" ledger "github.com/formancehq/ledger/internal" + "github.com/formancehq/ledger/pkg/features" "github.com/uptrace/bun" "go.opentelemetry.io/otel/trace" "text/template" @@ -66,8 +67,8 @@ func New(db *bun.DB, name string) *Bucket { } type ledgerSetup struct { - requireFeatures ledger.FeatureSet - script string + requireFeatures features.FeatureSet + script string } var ledgerSetups = []ledgerSetup{ @@ -96,14 +97,14 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", }, script: `create index "pcev_{{.ID}}" on "{{.Bucket}}".moves (accounts_address, asset, effective_date desc) where ledger = '{{.Name}}';`, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", }, script: ` create trigger "set_effective_volumes_{{.ID}}" @@ -117,8 +118,8 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", }, script: ` create trigger "update_effective_volumes_{{.ID}}" @@ -132,8 +133,8 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureHashLogs: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureHashLogs: "SYNC", }, script: ` create trigger "set_log_hash_{{.ID}}" @@ -147,8 +148,8 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureAccountMetadataHistory: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureAccountMetadataHistory: "SYNC", }, script: ` create trigger "update_account_metadata_history_{{.ID}}" @@ -162,8 +163,8 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureAccountMetadataHistory: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureAccountMetadataHistory: "SYNC", }, script: ` create trigger "insert_account_metadata_history_{{.ID}}" @@ -177,8 +178,8 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureTransactionMetadataHistory: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureTransactionMetadataHistory: "SYNC", }, script: ` create trigger "update_transaction_metadata_history_{{.ID}}" @@ -192,8 +193,8 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureTransactionMetadataHistory: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureTransactionMetadataHistory: "SYNC", }, script: ` create trigger "insert_transaction_metadata_history_{{.ID}}" @@ -207,24 +208,24 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexTransactionAccounts: "SYNC", + requireFeatures: features.FeatureSet{ + features.FeatureIndexTransactionAccounts: "SYNC", }, script: ` create index "transactions_sources_{{.ID}}" on "{{.Bucket}}".transactions using gin (sources jsonb_path_ops) where ledger = '{{.Name}}'; `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexTransactionAccounts: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexTransactionAccounts: "ON", }, script: ` create index "transactions_destinations_{{.ID}}" on "{{.Bucket}}".transactions using gin (destinations jsonb_path_ops) where ledger = '{{.Name}}'; `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexTransactionAccounts: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexTransactionAccounts: "ON", }, script: ` create trigger "transaction_set_addresses_{{.ID}}" @@ -238,24 +239,24 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexAddressSegments: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexAddressSegments: "ON", }, script: ` create index "accounts_address_array_{{.ID}}" on "{{.Bucket}}".accounts using gin (address_array jsonb_ops) where ledger = '{{.Name}}'; `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexAddressSegments: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexAddressSegments: "ON", }, script: ` create index "accounts_address_array_length_{{.ID}}" on "{{.Bucket}}".accounts (jsonb_array_length(address_array)) where ledger = '{{.Name}}'; `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexAddressSegments: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexAddressSegments: "ON", }, script: ` create trigger "accounts_set_address_array_{{.ID}}" @@ -269,27 +270,27 @@ var ledgerSetups = []ledgerSetup{ `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexAddressSegments: "ON", - ledger.FeatureIndexTransactionAccounts: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexAddressSegments: "ON", + features.FeatureIndexTransactionAccounts: "ON", }, script: ` create index "transactions_sources_arrays_{{.ID}}" on "{{.Bucket}}".transactions using gin (sources_arrays jsonb_path_ops) where ledger = '{{.Name}}'; `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexAddressSegments: "ON", - ledger.FeatureIndexTransactionAccounts: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexAddressSegments: "ON", + features.FeatureIndexTransactionAccounts: "ON", }, script: ` create index "transactions_destinations_arrays_{{.ID}}" on "{{.Bucket}}".transactions using gin (destinations_arrays jsonb_path_ops) where ledger = '{{.Name}}'; `, }, { - requireFeatures: ledger.FeatureSet{ - ledger.FeatureIndexAddressSegments: "ON", - ledger.FeatureIndexTransactionAccounts: "ON", + requireFeatures: features.FeatureSet{ + features.FeatureIndexAddressSegments: "ON", + features.FeatureIndexTransactionAccounts: "ON", }, script: ` create trigger "transaction_set_addresses_segments_{{.ID}}" @@ -302,4 +303,4 @@ var ledgerSetups = []ledgerSetup{ execute procedure "{{.Bucket}}".set_transaction_addresses_segments(); `, }, -} \ No newline at end of file +} diff --git a/internal/storage/ledger/accounts.go b/internal/storage/ledger/accounts.go index f95e1837d..1e34ca469 100644 --- a/internal/storage/ledger/accounts.go +++ b/internal/storage/ledger/accounts.go @@ -5,6 +5,7 @@ import ( "database/sql" "fmt" . "github.com/formancehq/go-libs/v2/bun/bunpaginate" + "github.com/formancehq/ledger/pkg/features" "regexp" "github.com/formancehq/ledger/internal/tracing" @@ -119,7 +120,7 @@ func (s *Store) selectAccounts(date *time.Time, expandVolumes, expandEffectiveVo ret = ret.Where("accounts.first_usage <= ?", date) } - if s.ledger.HasFeature(ledger.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { + if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { ret = ret. Join( `left join (?) accounts_metadata on accounts_metadata.accounts_address = accounts.address`, @@ -130,14 +131,14 @@ func (s *Store) selectAccounts(date *time.Time, expandVolumes, expandEffectiveVo ret = ret.ColumnExpr("accounts.metadata") } - if s.ledger.HasFeature(ledger.FeatureMovesHistory, "ON") && needVolumes { + if s.ledger.HasFeature(features.FeatureMovesHistory, "ON") && needVolumes { ret = ret.Join( `left join (?) volumes on volumes.accounts_address = accounts.address`, s.selectAccountWithAggregatedVolumes(date, true, "volumes"), ).Column("volumes.*") } - if s.ledger.HasFeature(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") && expandEffectiveVolumes { + if s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") && expandEffectiveVolumes { ret = ret.Join( `left join (?) effective_volumes on effective_volumes.accounts_address = accounts.address`, s.selectAccountWithAggregatedVolumes(date, false, "effective_volumes"), @@ -176,7 +177,7 @@ func (s *Store) selectAccounts(date *time.Time, expandVolumes, expandEffectiveVo String(), nil, nil case key == "metadata": - if s.ledger.HasFeature(ledger.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { + if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { key = "accounts_metadata.metadata" } @@ -184,7 +185,7 @@ func (s *Store) selectAccounts(date *time.Time, expandVolumes, expandEffectiveVo case metadataRegex.Match([]byte(key)): match := metadataRegex.FindAllStringSubmatch(key, 3) - if s.ledger.HasFeature(ledger.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { + if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { key = "accounts_metadata.metadata" } else { key = "metadata" diff --git a/internal/storage/ledger/balances.go b/internal/storage/ledger/balances.go index cd9f97a4e..442c984dc 100644 --- a/internal/storage/ledger/balances.go +++ b/internal/storage/ledger/balances.go @@ -3,6 +3,7 @@ package ledger import ( "context" "fmt" + "github.com/formancehq/ledger/pkg/features" "math/big" "strings" @@ -56,23 +57,23 @@ func (s *Store) selectAccountWithAssetAndVolumes(date *time.Time, useInsertionDa } } - if needAddressSegment && !s.ledger.HasFeature(ledger.FeatureIndexAddressSegments, "ON") { - return ret.Err(ledgercontroller.NewErrMissingFeature(ledger.FeatureIndexAddressSegments)) + if needAddressSegment && !s.ledger.HasFeature(features.FeatureIndexAddressSegments, "ON") { + return ret.Err(ledgercontroller.NewErrMissingFeature(features.FeatureIndexAddressSegments)) } var selectAccountsWithVolumes *bun.SelectQuery if date != nil && !date.IsZero() { if useInsertionDate { - if !s.ledger.HasFeature(ledger.FeatureMovesHistory, "ON") { - return ret.Err(ledgercontroller.NewErrMissingFeature(ledger.FeatureMovesHistory)) + if !s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + return ret.Err(ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory)) } selectAccountsWithVolumes = s.db.NewSelect(). TableExpr("(?) moves", s.SelectDistinctMovesBySeq(date)). Column("asset", "accounts_address"). ColumnExpr("post_commit_volumes as volumes") } else { - if !s.ledger.HasFeature(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { - return ret.Err(ledgercontroller.NewErrMissingFeature(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes)) + if !s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { + return ret.Err(ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes)) } selectAccountsWithVolumes = s.db.NewSelect(). TableExpr("(?) moves", s.SelectDistinctMovesByEffectiveDate(date)). @@ -92,7 +93,7 @@ func (s *Store) selectAccountWithAssetAndVolumes(date *time.Time, useInsertionDa TableExpr("(?) accounts_volumes", selectAccountsWithVolumes) if needMetadata { - if s.ledger.HasFeature(ledger.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { + if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { selectAccountsWithVolumes = selectAccountsWithVolumes. Join( `left join (?) accounts_metadata on accounts_metadata.accounts_address = accounts_volumes.accounts_address`, diff --git a/internal/storage/ledger/logs.go b/internal/storage/ledger/logs.go index 132ae8bfc..9d18352aa 100644 --- a/internal/storage/ledger/logs.go +++ b/internal/storage/ledger/logs.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "github.com/formancehq/ledger/internal/tracing" + "github.com/formancehq/ledger/pkg/features" "errors" "github.com/formancehq/go-libs/v2/bun/bunpaginate" @@ -55,7 +56,7 @@ func (s *Store) InsertLog(ctx context.Context, log *ledger.Log) error { tracing.NoResult(func(ctx context.Context) error { // We lock logs table as we need than the last log does not change until the transaction commit - if s.ledger.HasFeature(ledger.FeatureHashLogs, "SYNC") { + if s.ledger.HasFeature(features.FeatureHashLogs, "SYNC") { _, err := s.db.NewRaw(`select pg_advisory_xact_lock(?)`, s.ledger.ID).Exec(ctx) if err != nil { return postgres.ResolveError(err) diff --git a/internal/storage/ledger/moves.go b/internal/storage/ledger/moves.go index 7198cd9b5..2c32228c2 100644 --- a/internal/storage/ledger/moves.go +++ b/internal/storage/ledger/moves.go @@ -3,6 +3,7 @@ package ledger import ( "context" ledgercontroller "github.com/formancehq/ledger/internal/controller/ledger" + "github.com/formancehq/ledger/pkg/features" "github.com/formancehq/go-libs/v2/platform/postgres" "github.com/formancehq/go-libs/v2/time" @@ -14,8 +15,8 @@ import ( func (s *Store) SortMovesBySeq(date *time.Time) *bun.SelectQuery { ret := s.db.NewSelect() - if !s.ledger.HasFeature(ledger.FeatureMovesHistory, "ON") { - return ret.Err(ledgercontroller.NewErrMissingFeature(ledger.FeatureMovesHistory)) + if !s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + return ret.Err(ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory)) } ret = ret. diff --git a/internal/storage/ledger/store.go b/internal/storage/ledger/store.go index 26d2f6e05..ed9bfb6d3 100644 --- a/internal/storage/ledger/store.go +++ b/internal/storage/ledger/store.go @@ -6,6 +6,7 @@ import ( "github.com/formancehq/go-libs/v2/migrations" "github.com/formancehq/go-libs/v2/platform/postgres" "github.com/formancehq/ledger/internal/storage/bucket" + "github.com/formancehq/ledger/pkg/features" "go.opentelemetry.io/otel/metric" noopmetrics "go.opentelemetry.io/otel/metric/noop" "go.opentelemetry.io/otel/trace" @@ -71,8 +72,8 @@ func (s *Store) validateAddressFilter(operator string, value any) error { } if value, ok := value.(string); !ok { return fmt.Errorf("invalid 'address' filter") - } else if isSegmentedAddress(value) && !s.ledger.HasFeature(ledger.FeatureIndexAddressSegments, "ON") { - return fmt.Errorf("feature %s must be 'ON' to use segments address", ledger.FeatureIndexAddressSegments) + } else if isSegmentedAddress(value) && !s.ledger.HasFeature(features.FeatureIndexAddressSegments, "ON") { + return fmt.Errorf("feature %s must be 'ON' to use segments address", features.FeatureIndexAddressSegments) } return nil diff --git a/internal/storage/ledger/transactions.go b/internal/storage/ledger/transactions.go index 03aefb05c..6b49413b4 100644 --- a/internal/storage/ledger/transactions.go +++ b/internal/storage/ledger/transactions.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/formancehq/ledger/pkg/features" "math/big" "regexp" "slices" @@ -51,8 +52,8 @@ func (s *Store) selectDistinctTransactionMetadataHistories(date *time.Time) *bun func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffectiveVolumes bool, q query.Builder) *bun.SelectQuery { ret := s.db.NewSelect() - if expandEffectiveVolumes && !s.ledger.HasFeature(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { - return ret.Err(ledgercontroller.NewErrMissingFeature(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes)) + if expandEffectiveVolumes && !s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { + return ret.Err(ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes)) } if q != nil { @@ -116,7 +117,7 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti ret = ret.Where("timestamp <= ?", date) } - if s.ledger.HasFeature(ledger.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { + if s.ledger.HasFeature(features.FeatureAccountMetadataHistory, "SYNC") && date != nil && !date.IsZero() { ret = ret. Join( `left join (?) transactions_metadata on transactions_metadata.transactions_id = transactions.id`, @@ -127,7 +128,7 @@ func (s *Store) selectTransactions(date *time.Time, expandVolumes, expandEffecti ret = ret.ColumnExpr("metadata") } - if s.ledger.HasFeature(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") && expandEffectiveVolumes { + if s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") && expandEffectiveVolumes { ret = ret. Join( `join (?) pcev on pcev.transactions_id = transactions.id`, @@ -264,7 +265,7 @@ func (s *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) e } } - if s.ledger.HasFeature(ledger.FeatureMovesHistory, "ON") { + if s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { moves := ledger.Moves{} postings := tx.Postings slices.Reverse(postings) @@ -300,7 +301,7 @@ func (s *Store) CommitTransaction(ctx context.Context, tx *ledger.Transaction) e return fmt.Errorf("failed to insert moves: %w", err) } - if s.ledger.HasFeature(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { + if s.ledger.HasFeature(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "SYNC") { // todo: tx is inserted earlier! tx.PostCommitEffectiveVolumes = moves.ComputePostCommitEffectiveVolumes() } diff --git a/internal/storage/ledger/volumes.go b/internal/storage/ledger/volumes.go index aa8806108..c0a8644cf 100644 --- a/internal/storage/ledger/volumes.go +++ b/internal/storage/ledger/volumes.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/formancehq/go-libs/v2/collectionutils" "github.com/formancehq/go-libs/v2/platform/postgres" + "github.com/formancehq/ledger/pkg/features" "github.com/formancehq/ledger/internal/tracing" @@ -67,8 +68,8 @@ func (s *Store) UpdateVolumes(ctx context.Context, accountVolumes ...ledger.Acco func (s *Store) selectVolumes(oot, pit *time.Time, useInsertionDate bool, groupLevel int, q lquery.Builder) *bun.SelectQuery { ret := s.db.NewSelect() - if !s.ledger.HasFeature(ledger.FeatureMovesHistory, "ON") { - return ret.Err(ledgercontroller.NewErrMissingFeature(ledger.FeatureMovesHistory)) + if !s.ledger.HasFeature(features.FeatureMovesHistory, "ON") { + return ret.Err(ledgercontroller.NewErrMissingFeature(features.FeatureMovesHistory)) } var ( diff --git a/internal/storage/module.go b/internal/storage/module.go index b0aa35370..fd524c8f0 100644 --- a/internal/storage/module.go +++ b/internal/storage/module.go @@ -16,9 +16,9 @@ func NewFXModule(autoUpgrade bool) fx.Option { ret = append(ret, fx.Invoke(func(lc fx.Lifecycle, driver *driver.Driver) { var ( - upgradeContext context.Context - cancelContext func() - upgradeStopped = make(chan struct{}) + upgradeContext context.Context + cancelContext func() + upgradeStopped = make(chan struct{}) minimalVersionReached = make(chan struct{}) ) lc.Append(fx.Hook{ @@ -77,4 +77,4 @@ func migrate(ctx context.Context, driver *driver.Driver, minimalVersionReached c return } } -} \ No newline at end of file +} diff --git a/openapi.yaml b/openapi.yaml index 81042b8ec..181026535 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1208,6 +1208,30 @@ paths: security: - Authorization: - ledger:read + /_/metrics: + get: + tags: + - ledger + summary: Read in memory metrics + operationId: getMetrics + x-speakeasy-name-override: GetMetrics + responses: + "200": + description: OK + content: + application/json: + schema: + type: object + additionalProperties: {} + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/V2ErrorResponse' + security: + - Authorization: + - ledger:read /v2: get: summary: List ledgers diff --git a/openapi/v2.yaml b/openapi/v2.yaml index 137eb09c8..194e2c331 100644 --- a/openapi/v2.yaml +++ b/openapi/v2.yaml @@ -35,6 +35,30 @@ paths: security: - Authorization: - ledger:read + /_/metrics: + get: + tags: + - ledger + summary: Read in memory metrics + operationId: getMetrics + x-speakeasy-name-override: GetMetrics + responses: + "200": + description: OK + content: + application/json: + schema: + type: object + additionalProperties: {} + default: + description: Error + content: + application/json: + schema: + $ref: "#/components/schemas/V2ErrorResponse" + security: + - Authorization: + - ledger:read /v2: get: summary: List ledgers diff --git a/pkg/client/.speakeasy/gen.lock b/pkg/client/.speakeasy/gen.lock index 19f5d974f..960ab7ad2 100644 --- a/pkg/client/.speakeasy/gen.lock +++ b/pkg/client/.speakeasy/gen.lock @@ -1,12 +1,12 @@ lockVersion: 2.0.0 id: a9ac79e1-e429-4ee3-96c4-ec973f19bec3 management: - docChecksum: 5e24fc96851e508606f6b6668ed3ffb3 + docChecksum: 743c41071ff98ac5e7d00e58f65650e3 docVersion: v1 speakeasyVersion: 1.351.0 generationVersion: 2.384.1 - releaseVersion: 0.4.30 - configChecksum: b752be0bf02f575b11e046541aa57005 + releaseVersion: 0.4.31 + configChecksum: 006527a5b70ea037906b0ff29a01a2ef features: go: additionalDependencies: 0.1.0 @@ -55,6 +55,7 @@ generatedFiles: - internal/utils/security.go - internal/utils/utils.go - /models/operations/v2getinfo.go + - /models/operations/getmetrics.go - /models/operations/getinfo.go - /models/operations/getledgerinfo.go - /models/operations/countaccounts.go @@ -173,6 +174,7 @@ generatedFiles: - /models/components/v2log.go - /models/components/security.go - docs/models/operations/v2getinforesponse.md + - docs/models/operations/getmetricsresponse.md - docs/models/operations/getinforesponse.md - docs/models/operations/getledgerinforequest.md - docs/models/operations/getledgerinforesponse.md diff --git a/pkg/client/.speakeasy/gen.yaml b/pkg/client/.speakeasy/gen.yaml index 93655e172..965163ad3 100644 --- a/pkg/client/.speakeasy/gen.yaml +++ b/pkg/client/.speakeasy/gen.yaml @@ -15,7 +15,7 @@ generation: auth: oAuth2ClientCredentialsEnabled: true go: - version: 0.4.30 + version: 0.4.31 additionalDependencies: {} allowUnknownFieldsInWeakUnions: false clientServerStatusCodesAsErrors: true diff --git a/pkg/client/README.md b/pkg/client/README.md index 4d6bc2c4c..613553c69 100644 --- a/pkg/client/README.md +++ b/pkg/client/README.md @@ -87,6 +87,7 @@ func main() { ### [Ledger](docs/sdks/ledger/README.md) * [GetInfo](docs/sdks/ledger/README.md#getinfo) - Show server information +* [GetMetrics](docs/sdks/ledger/README.md#getmetrics) - Read in memory metrics ### [Ledger.V1](docs/sdks/v1/README.md) diff --git a/pkg/client/docs/models/operations/getmetricsresponse.md b/pkg/client/docs/models/operations/getmetricsresponse.md new file mode 100644 index 000000000..721394c72 --- /dev/null +++ b/pkg/client/docs/models/operations/getmetricsresponse.md @@ -0,0 +1,9 @@ +# GetMetricsResponse + + +## Fields + +| Field | Type | Required | Description | +| ------------------------------------------------------------------ | ------------------------------------------------------------------ | ------------------------------------------------------------------ | ------------------------------------------------------------------ | +| `HTTPMeta` | [components.HTTPMetadata](../../models/components/httpmetadata.md) | :heavy_check_mark: | N/A | +| `Object` | map[string]*any* | :heavy_minus_sign: | OK | \ No newline at end of file diff --git a/pkg/client/docs/sdks/ledger/README.md b/pkg/client/docs/sdks/ledger/README.md index a0d4fc64e..4c46b7a04 100644 --- a/pkg/client/docs/sdks/ledger/README.md +++ b/pkg/client/docs/sdks/ledger/README.md @@ -4,6 +4,7 @@ ### Available Operations * [GetInfo](#getinfo) - Show server information +* [GetMetrics](#getmetrics) - Read in memory metrics ## GetInfo @@ -55,3 +56,54 @@ func main() { | ------------------------- | ------------------------- | ------------------------- | | sdkerrors.V2ErrorResponse | default | application/json | | sdkerrors.SDKError | 4xx-5xx | */* | + +## GetMetrics + +Read in memory metrics + +### Example Usage + +```go +package main + +import( + "github.com/formancehq/ledger/pkg/client/models/components" + "github.com/formancehq/ledger/pkg/client" + "context" + "log" +) + +func main() { + s := client.New( + client.WithSecurity(components.Security{ + ClientID: "", + ClientSecret: "", + }), + ) + + ctx := context.Background() + res, err := s.Ledger.GetMetrics(ctx) + if err != nil { + log.Fatal(err) + } + if res.Object != nil { + // handle response + } +} +``` + +### Parameters + +| Parameter | Type | Required | Description | +| -------------------------------------------------------- | -------------------------------------------------------- | -------------------------------------------------------- | -------------------------------------------------------- | +| `ctx` | [context.Context](https://pkg.go.dev/context#Context) | :heavy_check_mark: | The context to use for the request. | +| `opts` | [][operations.Option](../../models/operations/option.md) | :heavy_minus_sign: | The options for this request. | + + +### Response + +**[*operations.GetMetricsResponse](../../models/operations/getmetricsresponse.md), error** +| Error Object | Status Code | Content Type | +| ------------------------- | ------------------------- | ------------------------- | +| sdkerrors.V2ErrorResponse | default | application/json | +| sdkerrors.SDKError | 4xx-5xx | */* | diff --git a/pkg/client/formance.go b/pkg/client/formance.go index 11a85fa6d..b6ec8384d 100644 --- a/pkg/client/formance.go +++ b/pkg/client/formance.go @@ -143,9 +143,9 @@ func New(opts ...SDKOption) *Formance { sdkConfiguration: sdkConfiguration{ Language: "go", OpenAPIDocVersion: "v1", - SDKVersion: "0.4.30", + SDKVersion: "0.4.31", GenVersion: "2.384.1", - UserAgent: "speakeasy-sdk/go 0.4.30 2.384.1 v1 github.com/formancehq/ledger/pkg/client", + UserAgent: "speakeasy-sdk/go 0.4.31 2.384.1 v1 github.com/formancehq/ledger/pkg/client", Hooks: hooks.New(), }, } diff --git a/pkg/client/ledger.go b/pkg/client/ledger.go index 621bbfe98..a3af51ae0 100644 --- a/pkg/client/ledger.go +++ b/pkg/client/ledger.go @@ -222,3 +222,182 @@ func (s *Ledger) GetInfo(ctx context.Context, opts ...operations.Option) (*opera return res, nil } + +// GetMetrics - Read in memory metrics +func (s *Ledger) GetMetrics(ctx context.Context, opts ...operations.Option) (*operations.GetMetricsResponse, error) { + hookCtx := hooks.HookContext{ + Context: ctx, + OperationID: "getMetrics", + OAuth2Scopes: []string{"ledger:read", "ledger:read"}, + SecuritySource: s.sdkConfiguration.Security, + } + + o := operations.Options{} + supportedOptions := []string{ + operations.SupportedOptionRetries, + operations.SupportedOptionTimeout, + } + + for _, opt := range opts { + if err := opt(&o, supportedOptions...); err != nil { + return nil, fmt.Errorf("error applying option: %w", err) + } + } + + baseURL := utils.ReplaceParameters(s.sdkConfiguration.GetServerDetails()) + opURL, err := url.JoinPath(baseURL, "/_/metrics") + if err != nil { + return nil, fmt.Errorf("error generating URL: %w", err) + } + + timeout := o.Timeout + if timeout == nil { + timeout = s.sdkConfiguration.Timeout + } + + if timeout != nil { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, *timeout) + defer cancel() + } + + req, err := http.NewRequestWithContext(ctx, "GET", opURL, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %w", err) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", s.sdkConfiguration.UserAgent) + + if err := utils.PopulateSecurity(ctx, req, s.sdkConfiguration.Security); err != nil { + return nil, err + } + + globalRetryConfig := s.sdkConfiguration.RetryConfig + retryConfig := o.Retries + if retryConfig == nil { + if globalRetryConfig != nil { + retryConfig = globalRetryConfig + } + } + + var httpRes *http.Response + if retryConfig != nil { + httpRes, err = utils.Retry(ctx, utils.Retries{ + Config: retryConfig, + StatusCodes: []string{ + "429", + "500", + "502", + "503", + "504", + }, + }, func() (*http.Response, error) { + if req.Body != nil { + copyBody, err := req.GetBody() + if err != nil { + return nil, err + } + req.Body = copyBody + } + + req, err = s.sdkConfiguration.Hooks.BeforeRequest(hooks.BeforeRequestContext{HookContext: hookCtx}, req) + if err != nil { + return nil, backoff.Permanent(err) + } + + httpRes, err := s.sdkConfiguration.Client.Do(req) + if err != nil || httpRes == nil { + if err != nil { + err = fmt.Errorf("error sending request: %w", err) + } else { + err = fmt.Errorf("error sending request: no response") + } + + _, err = s.sdkConfiguration.Hooks.AfterError(hooks.AfterErrorContext{HookContext: hookCtx}, nil, err) + } + return httpRes, err + }) + + if err != nil { + return nil, err + } else { + httpRes, err = s.sdkConfiguration.Hooks.AfterSuccess(hooks.AfterSuccessContext{HookContext: hookCtx}, httpRes) + if err != nil { + return nil, err + } + } + } else { + req, err = s.sdkConfiguration.Hooks.BeforeRequest(hooks.BeforeRequestContext{HookContext: hookCtx}, req) + if err != nil { + return nil, err + } + + httpRes, err = s.sdkConfiguration.Client.Do(req) + if err != nil || httpRes == nil { + if err != nil { + err = fmt.Errorf("error sending request: %w", err) + } else { + err = fmt.Errorf("error sending request: no response") + } + + _, err = s.sdkConfiguration.Hooks.AfterError(hooks.AfterErrorContext{HookContext: hookCtx}, nil, err) + return nil, err + } else if utils.MatchStatusCodes([]string{"default"}, httpRes.StatusCode) { + _httpRes, err := s.sdkConfiguration.Hooks.AfterError(hooks.AfterErrorContext{HookContext: hookCtx}, httpRes, nil) + if err != nil { + return nil, err + } else if _httpRes != nil { + httpRes = _httpRes + } + } else { + httpRes, err = s.sdkConfiguration.Hooks.AfterSuccess(hooks.AfterSuccessContext{HookContext: hookCtx}, httpRes) + if err != nil { + return nil, err + } + } + } + + res := &operations.GetMetricsResponse{ + HTTPMeta: components.HTTPMetadata{ + Request: req, + Response: httpRes, + }, + } + + rawBody, err := io.ReadAll(httpRes.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %w", err) + } + httpRes.Body.Close() + httpRes.Body = io.NopCloser(bytes.NewBuffer(rawBody)) + + switch { + case httpRes.StatusCode == 200: + switch { + case utils.MatchContentType(httpRes.Header.Get("Content-Type"), `application/json`): + var out map[string]any + if err := utils.UnmarshalJsonFromResponseBody(bytes.NewBuffer(rawBody), &out, ""); err != nil { + return nil, err + } + + res.Object = out + default: + return nil, sdkerrors.NewSDKError(fmt.Sprintf("unknown content-type received: %s", httpRes.Header.Get("Content-Type")), httpRes.StatusCode, string(rawBody), httpRes) + } + default: + switch { + case utils.MatchContentType(httpRes.Header.Get("Content-Type"), `application/json`): + var out sdkerrors.V2ErrorResponse + if err := utils.UnmarshalJsonFromResponseBody(bytes.NewBuffer(rawBody), &out, ""); err != nil { + return nil, err + } + + return nil, &out + default: + return nil, sdkerrors.NewSDKError(fmt.Sprintf("unknown content-type received: %s", httpRes.Header.Get("Content-Type")), httpRes.StatusCode, string(rawBody), httpRes) + } + } + + return res, nil + +} diff --git a/pkg/client/models/operations/getmetrics.go b/pkg/client/models/operations/getmetrics.go new file mode 100644 index 000000000..ea68147d5 --- /dev/null +++ b/pkg/client/models/operations/getmetrics.go @@ -0,0 +1,27 @@ +// Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT. + +package operations + +import ( + "github.com/formancehq/ledger/pkg/client/models/components" +) + +type GetMetricsResponse struct { + HTTPMeta components.HTTPMetadata `json:"-"` + // OK + Object map[string]any +} + +func (o *GetMetricsResponse) GetHTTPMeta() components.HTTPMetadata { + if o == nil { + return components.HTTPMetadata{} + } + return o.HTTPMeta +} + +func (o *GetMetricsResponse) GetObject() map[string]any { + if o == nil { + return nil + } + return o.Object +} diff --git a/pkg/features/features.go b/pkg/features/features.go new file mode 100644 index 000000000..bf8bb401d --- /dev/null +++ b/pkg/features/features.go @@ -0,0 +1,119 @@ +package features + +import ( + "fmt" + "github.com/formancehq/go-libs/v2/collectionutils" + "slices" + "strings" +) + +const ( + // FeatureMovesHistory is used to define if the ledger has to save funds movements history. + // Value is either ON or OFF + FeatureMovesHistory = "MOVES_HISTORY" + // FeatureMovesHistoryPostCommitEffectiveVolumes is used to define if the pvce property of funds movements history + // has to be updated with back dated transaction. + // Value is either SYNC or DISABLED. + // todo: depends on FeatureMovesHistory (dependency should be checked) + FeatureMovesHistoryPostCommitEffectiveVolumes = "MOVES_HISTORY_POST_COMMIT_EFFECTIVE_VOLUMES" + // FeatureHashLogs is used to defined it the logs has to be hashed. + FeatureHashLogs = "HASH_LOGS" + // FeatureAccountMetadataHistory is used to defined it the account metadata must be historized. + FeatureAccountMetadataHistory = "ACCOUNT_METADATA_HISTORY" + // FeatureTransactionMetadataHistory is used to defined it the transaction metadata must be historized. + FeatureTransactionMetadataHistory = "TRANSACTION_METADATA_HISTORY" + // FeatureIndexAddressSegments is used to defined it we want to index segments of accounts address. + // Without this feature, the ledger will not allow filtering on partial account address. + FeatureIndexAddressSegments = "INDEX_ADDRESS_SEGMENTS" + // FeatureIndexTransactionAccounts is used to defined it we want to index accounts used in a transaction. + FeatureIndexTransactionAccounts = "INDEX_TRANSACTION_ACCOUNTS" +) + +var ( + DefaultFeatures = FeatureSet{ + FeatureMovesHistory: "ON", + FeatureMovesHistoryPostCommitEffectiveVolumes: "SYNC", + FeatureHashLogs: "SYNC", + FeatureAccountMetadataHistory: "SYNC", + FeatureTransactionMetadataHistory: "SYNC", + FeatureIndexAddressSegments: "ON", + FeatureIndexTransactionAccounts: "ON", + } + MinimalFeatureSet = FeatureSet{ + FeatureMovesHistory: "OFF", + FeatureMovesHistoryPostCommitEffectiveVolumes: "DISABLED", + FeatureHashLogs: "DISABLED", + FeatureAccountMetadataHistory: "DISABLED", + FeatureTransactionMetadataHistory: "DISABLED", + FeatureIndexAddressSegments: "OFF", + FeatureIndexTransactionAccounts: "OFF", + } + FeatureConfigurations = map[string][]string{ + FeatureMovesHistory: {"ON", "OFF"}, + FeatureMovesHistoryPostCommitEffectiveVolumes: {"SYNC", "DISABLED"}, + FeatureHashLogs: {"SYNC", "DISABLED"}, + FeatureAccountMetadataHistory: {"SYNC", "DISABLED"}, + FeatureTransactionMetadataHistory: {"SYNC", "DISABLED"}, + FeatureIndexAddressSegments: {"ON", "OFF"}, + FeatureIndexTransactionAccounts: {"ON", "OFF"}, + } +) + +func ValidateFeatureWithValue(feature, value string) error { + possibleConfigurations, ok := FeatureConfigurations[feature] + if !ok { + return fmt.Errorf("feature %q not exists", feature) + } + if !slices.Contains(possibleConfigurations, value) { + return fmt.Errorf("configuration %s it not possible for feature %s", value, feature) + } + + return nil +} + +type FeatureSet map[string]string + +func (f FeatureSet) With(feature, value string) FeatureSet { + ret := FeatureSet{} + for k, v := range f { + ret[k] = v + } + ret[feature] = value + + return ret +} + +func (f FeatureSet) SortedKeys() []string { + ret := collectionutils.Keys(f) + slices.Sort(ret) + + return ret +} + +func (f FeatureSet) String() string { + if len(f) == 0 { + return "" + } + + ret := "" + for _, key := range f.SortedKeys() { + ret = ret + "," + shortenFeature(key) + "=" + f[key] + } + + return ret[1:] +} + +func (f FeatureSet) Match(features FeatureSet) bool { + for k, v := range features { + if f[k] != v { + return false + } + } + return true +} + +func shortenFeature(feature string) string { + return strings.Join(collectionutils.Map(strings.Split(feature, "_"), func(from string) string { + return from[:1] + }), "") +} diff --git a/pkg/generate/generator.go b/pkg/generate/generator.go index 11a718c0c..5681c6950 100644 --- a/pkg/generate/generator.go +++ b/pkg/generate/generator.go @@ -191,11 +191,6 @@ func NewGenerator(script string, opts ...Option) (*Generator, error) { runtime := goja.New() - _, err := runtime.RunString(script) - if err != nil { - return nil, err - } - for k, v := range cfg.globals { err := runtime.Set(k, v) if err != nil { @@ -203,6 +198,11 @@ func NewGenerator(script string, opts ...Option) (*Generator, error) { } } + _, err := runtime.RunString(script) + if err != nil { + return nil, err + } + runtime.SetFieldNameMapper(goja.TagFieldNameMapper("json", true)) err = runtime.Set("uuid", uuid.NewString) diff --git a/pkg/generate/set.go b/pkg/generate/set.go new file mode 100644 index 000000000..b3ae52f63 --- /dev/null +++ b/pkg/generate/set.go @@ -0,0 +1,73 @@ +package generate + +import ( + "context" + "errors" + "fmt" + "github.com/formancehq/go-libs/v2/logging" + "github.com/formancehq/ledger/pkg/client" + "golang.org/x/sync/errgroup" +) + +type GeneratorSet struct { + vus int + script string + targetedLedger string + client *client.Formance + untilLogID uint64 +} + +func (s *GeneratorSet) Run(ctx context.Context) error { + parallelContext, cancel := context.WithCancel(ctx) + defer cancel() + + errGroup, ctx := errgroup.WithContext(parallelContext) + + for vu := 0; vu < s.vus; vu++ { + generator, err := NewGenerator(s.script, WithGlobals(map[string]any{ + "vu": vu, + })) + if err != nil { + return fmt.Errorf("failed to create generator: %w", err) + } + + errGroup.Go(func() error { + defer cancel() + + iteration := 0 + + for { + logging.FromContext(ctx).Debugf("Run iteration %d/%d", vu, iteration) + + action, err := generator.Next(vu) + if err != nil { + return fmt.Errorf("iteration %d/%d failed: %w", vu, iteration, err) + } + + ret, err := action.Apply(ctx, s.client.Ledger.V2, s.targetedLedger) + if err != nil { + if errors.Is(err, context.Canceled) { + return nil + } + return fmt.Errorf("iteration %d/%d failed: %w", vu, iteration, err) + } + if s.untilLogID != 0 && uint64(ret.GetLogID()) >= s.untilLogID { + return nil + } + iteration++ + } + }) + } + + return errGroup.Wait() +} + +func NewGeneratorSet(vus int, script string, targetedLedger string, client *client.Formance, untilLogID uint64) *GeneratorSet { + return &GeneratorSet{ + vus: vus, + script: script, + targetedLedger: targetedLedger, + client: client, + untilLogID: untilLogID, + } +} diff --git a/pkg/testserver/server.go b/pkg/testserver/server.go index 45b1046d0..c479887a9 100644 --- a/pkg/testserver/server.go +++ b/pkg/testserver/server.go @@ -45,7 +45,7 @@ type Configuration struct { Debug bool OTLPConfig *OTLPConfig ExperimentalFeatures bool - DisableAutoUpgrade bool + DisableAutoUpgrade bool BulkMaxSize int ExperimentalNumscriptRewrite bool } diff --git a/test/e2e/api_ledgers_create_test.go b/test/e2e/api_ledgers_create_test.go index 9aedb0030..9242dcf32 100644 --- a/test/e2e/api_ledgers_create_test.go +++ b/test/e2e/api_ledgers_create_test.go @@ -6,9 +6,9 @@ import ( "github.com/formancehq/go-libs/v2/logging" "github.com/formancehq/go-libs/v2/pointer" . "github.com/formancehq/go-libs/v2/testing/api" - ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/ledger/pkg/client/models/components" "github.com/formancehq/ledger/pkg/client/models/operations" + "github.com/formancehq/ledger/pkg/features" . "github.com/formancehq/ledger/pkg/testserver" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -49,8 +49,8 @@ var _ = Context("Ledger engine tests", func() { }) Context("with specific features set", func() { BeforeEach(func() { - createLedgerRequest.V2CreateLedgerRequest.Features = ledger.MinimalFeatureSet. - With(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes, "DISABLED") + createLedgerRequest.V2CreateLedgerRequest.Features = features.MinimalFeatureSet. + With(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "DISABLED") }) It("should be ok", func() { Expect(err).To(BeNil()) @@ -58,8 +58,8 @@ var _ = Context("Ledger engine tests", func() { }) Context("with invalid feature configuration", func() { BeforeEach(func() { - createLedgerRequest.V2CreateLedgerRequest.Features = ledger.MinimalFeatureSet. - With(ledger.FeatureMovesHistoryPostCommitEffectiveVolumes, "XXX") + createLedgerRequest.V2CreateLedgerRequest.Features = features.MinimalFeatureSet. + With(features.FeatureMovesHistoryPostCommitEffectiveVolumes, "XXX") }) It("should fail", func() { Expect(err).To(HaveErrorCode(string(components.V2ErrorsEnumValidation))) @@ -67,7 +67,7 @@ var _ = Context("Ledger engine tests", func() { }) Context("with invalid feature name", func() { BeforeEach(func() { - createLedgerRequest.V2CreateLedgerRequest.Features = ledger.MinimalFeatureSet. + createLedgerRequest.V2CreateLedgerRequest.Features = features.MinimalFeatureSet. With("foo", "XXX") }) It("should fail", func() { diff --git a/test/e2e/api_ledgers_import_test.go b/test/e2e/api_ledgers_import_test.go index 6f3aca9c9..e3830d6c8 100644 --- a/test/e2e/api_ledgers_import_test.go +++ b/test/e2e/api_ledgers_import_test.go @@ -5,12 +5,12 @@ package test_suite import ( "database/sql" . "github.com/formancehq/go-libs/v2/testing/api" + "github.com/formancehq/ledger/pkg/features" "io" "math/big" "github.com/formancehq/go-libs/v2/logging" "github.com/formancehq/go-libs/v2/pointer" - ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/ledger/pkg/client/models/components" "github.com/formancehq/ledger/pkg/client/models/operations" . "github.com/formancehq/ledger/pkg/testserver" @@ -43,7 +43,7 @@ var _ = Context("Ledger engine tests", func() { createLedgerRequest = operations.V2CreateLedgerRequest{ Ledger: "foo", V2CreateLedgerRequest: &components.V2CreateLedgerRequest{ - Features: ledger.MinimalFeatureSet, + Features: features.MinimalFeatureSet, }, } }) @@ -121,7 +121,7 @@ var _ = Context("Ledger engine tests", func() { err := CreateLedger(ctx, testServer.GetValue(), operations.V2CreateLedgerRequest{ Ledger: ledgerCopyName, V2CreateLedgerRequest: &components.V2CreateLedgerRequest{ - Features: ledger.MinimalFeatureSet, + Features: features.MinimalFeatureSet, }, }) Expect(err).To(BeNil()) diff --git a/test/performance/benchmark_test.go b/test/performance/benchmark_test.go index 2182d827c..8f3f6c5d4 100644 --- a/test/performance/benchmark_test.go +++ b/test/performance/benchmark_test.go @@ -4,11 +4,9 @@ package performance_test import ( "context" - "encoding/json" "fmt" . "github.com/formancehq/go-libs/v2/collectionutils" "github.com/formancehq/ledger/pkg/generate" - "net/http" "sort" "sync/atomic" "testing" @@ -115,14 +113,11 @@ func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Result { report.End = time.Now() // Fetch otel metrics - rsp, err := http.Get(env.URL() + "/_/metrics") - require.NoError(b, err) - if rsp.StatusCode == http.StatusOK { - ret := make(map[string]any) - require.NoError(b, json.NewDecoder(rsp.Body).Decode(&ret)) - report.InternalMetrics = ret + metrics, err := env.Client().Ledger.GetMetrics(ctx) + if err != nil { + b.Logf("Unable to fetch ledger metrics: %s", err) } else { - b.Logf("Unable to fetch ledger metrics, got status code %d", rsp.StatusCode) + report.InternalMetrics = metrics.Object } // Compute final results diff --git a/test/performance/features_test.go b/test/performance/features_test.go index a3afc7c02..299094a27 100644 --- a/test/performance/features_test.go +++ b/test/performance/features_test.go @@ -4,7 +4,7 @@ package performance_test import ( . "github.com/formancehq/go-libs/v2/collectionutils" - ledger "github.com/formancehq/ledger/internal" + features2 "github.com/formancehq/ledger/pkg/features" "sort" ) @@ -12,19 +12,19 @@ func buildAllPossibleConfigurations() []configuration { possibleConfigurations := make([]configuration, 0) possibleConfigurations = append(possibleConfigurations, configuration{ Name: "MINIMAL", - FeatureSet: ledger.MinimalFeatureSet, + FeatureSet: features2.MinimalFeatureSet, }) - fullConfiguration := ledger.MinimalFeatureSet - features := Keys(ledger.FeatureConfigurations) + fullConfiguration := features2.MinimalFeatureSet + features := Keys(features2.FeatureConfigurations) sort.Strings(features) for _, feature := range features { possibleConfigurations = append(possibleConfigurations, configuration{ Name: feature, - FeatureSet: ledger.MinimalFeatureSet.With(feature, ledger.FeatureConfigurations[feature][0]), + FeatureSet: features2.MinimalFeatureSet.With(feature, features2.FeatureConfigurations[feature][0]), }) - fullConfiguration = fullConfiguration.With(feature, ledger.FeatureConfigurations[feature][0]) + fullConfiguration = fullConfiguration.With(feature, features2.FeatureConfigurations[feature][0]) } possibleConfigurations = append(possibleConfigurations, configuration{ Name: "FULL", @@ -36,7 +36,7 @@ func buildAllPossibleConfigurations() []configuration { type configuration struct { Name string - FeatureSet ledger.FeatureSet + FeatureSet features2.FeatureSet } func (c configuration) String() string { diff --git a/test/rolling-upgrades/main_test.go b/test/rolling-upgrades/main_test.go index bfa92f7b9..c4ba2f72f 100644 --- a/test/rolling-upgrades/main_test.go +++ b/test/rolling-upgrades/main_test.go @@ -5,7 +5,7 @@ import ( "flag" "fmt" "github.com/formancehq/go-libs/v2/logging" - ledger "github.com/formancehq/ledger/internal" + "github.com/formancehq/ledger/pkg/features" corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3" metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" @@ -197,10 +197,10 @@ func deployTest(ctx *pulumi.Context) error { pulumi.String("30"), } - for _, key := range ledger.MinimalFeatureSet.SortedKeys() { + for _, key := range features.MinimalFeatureSet.SortedKeys() { generatorArgs = append(generatorArgs, pulumi.String("--ledger-feature"), - pulumi.String(key+"="+ledger.MinimalFeatureSet[key]), + pulumi.String(key+"="+features.MinimalFeatureSet[key]), ) } diff --git a/test/stress/stress_test.go b/test/stress/stress_test.go index 68f25c1b1..23117440e 100644 --- a/test/stress/stress_test.go +++ b/test/stress/stress_test.go @@ -4,6 +4,7 @@ package test_suite import ( "fmt" + "github.com/formancehq/ledger/pkg/features" "math/big" "math/rand" "sync" @@ -13,7 +14,6 @@ import ( "github.com/formancehq/go-libs/v2/logging" "github.com/formancehq/go-libs/v2/pointer" "github.com/formancehq/go-libs/v2/testing/platform/pgtesting" - ledger "github.com/formancehq/ledger/internal" "github.com/formancehq/ledger/pkg/client/models/components" "github.com/formancehq/ledger/pkg/client/models/operations" . "github.com/formancehq/ledger/pkg/testserver" @@ -52,7 +52,7 @@ var _ = Context("Ledger stress tests", func() { Ledger: ledgerName, V2CreateLedgerRequest: &components.V2CreateLedgerRequest{ Bucket: &bucketName, - Features: ledger.MinimalFeatureSet.With(ledger.FeatureMovesHistory, "ON"), + Features: features.MinimalFeatureSet.With(features.FeatureMovesHistory, "ON"), }, }) Expect(err).ShouldNot(HaveOccurred()) diff --git a/tools/generator/cmd/root.go b/tools/generator/cmd/root.go index be4fae102..158d66770 100644 --- a/tools/generator/cmd/root.go +++ b/tools/generator/cmd/root.go @@ -14,7 +14,6 @@ import ( "github.com/spf13/cobra" "golang.org/x/oauth2" "golang.org/x/oauth2/clientcredentials" - "golang.org/x/sync/errgroup" "net/http" "os" "strings" @@ -163,49 +162,11 @@ func run(cmd *cobra.Command, args []string) error { } } - parallelContext, cancel := context.WithCancel(cmd.Context()) - defer cancel() - - errGroup, ctx := errgroup.WithContext(parallelContext) - logging.FromContext(cmd.Context()).Infof("Starting to generate data with %d vus", vus) - for vu := 0; vu < vus; vu++ { - generator, err := generate.NewGenerator(string(fileContent), generate.WithGlobals(map[string]any{ - "vu": vu, - })) - if err != nil { - return fmt.Errorf("failed to create generator: %w", err) - } - - errGroup.Go(func() error { - defer cancel() - - iteration := 0 - - for { - logging.FromContext(ctx).Infof("Run iteration %d/%d", vu, iteration) - action, err := generator.Next(vu) - if err != nil { - return fmt.Errorf("iteration %d/%d failed: %w", vu, iteration, err) - } - - ret, err := action.Apply(ctx, client.Ledger.V2, targetedLedger) - if err != nil { - if errors.Is(err, context.Canceled) { - return nil - } - return fmt.Errorf("iteration %d/%d failed: %w", vu, iteration, err) - } - if untilLogID != 0 && ret.GetLogID() >= untilLogID { - return nil - } - iteration++ - } - }) - } - - return errGroup.Wait() + return generate. + NewGeneratorSet(vus, string(fileContent), targetedLedger, client, uint64(untilLogID)). + Run(cmd.Context()) } func extractSliceSliceFlag(cmd *cobra.Command, flagName string) (map[string]string, error) {