From 8937067248ad32edea5a7a876007b7fb36ff0b8e Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Fri, 4 Oct 2024 15:47:21 +0200 Subject: [PATCH] feat: refine benchmark and add plot script --- .gitignore | 1 + CONTRIBUTING.md | 4 +- go.mod | 2 + go.sum | 4 + internal/ledger.go | 23 ++++++ test/performance/README.md | 81 +++++--------------- test/performance/benchmark_test.go | 18 ++--- test/performance/features_test.go | 74 +++++++----------- test/performance/main_test.go | 29 ++++++- test/performance/plot/features_comparison.gp | 12 +++ test/performance/report_test.go | 24 ++---- test/performance/write_test.go | 65 +++++++++++----- 12 files changed, 176 insertions(+), 161 deletions(-) create mode 100644 test/performance/plot/features_comparison.gp diff --git a/.gitignore b/.gitignore index 3f5e09280..a8743234f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ cover.out go.work* *.jar +report \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e8100f5d..b73b54152 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,7 +121,7 @@ Additionally, each ledger is created on a bucket, `quickstart` is installed on b > > But, automatically created ledgers by v1 api will create a new bucket with the same name as the ledger. That's for compatibility reasons regarding ledger v1 behavior. -***Buckets*** +### Buckets To create a ledger on a specific bucket, use the command: @@ -150,7 +150,7 @@ $ curl http://localhost:3068/v2/testing | jq Under the hood, a bucket is a Postgres schema. You can use the bucket feature to implement some kind of horizontal scaling. -***Features*** +### Features Each usage of the ledger service, is different. Some usage involve a high write throughput, some other involve high read throughput, custom aggregation etc... diff --git a/go.mod b/go.mod index 9c370e748..ee214dd31 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( require ( dario.cat/mergo v1.0.1 // indirect filippo.io/edwards25519 v1.1.0 // indirect + github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/IBM/sarama v1.43.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -86,6 +87,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-chi/render v1.0.3 // indirect + github.com/go-echarts/go-echarts/v2 v2.4.2 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect diff --git a/go.sum b/go.sum index 311c9b34e..f760ea3b1 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4 h1:tTl76UtV0KnHdcIxHAhq31NNO/B/r0+LnPsvUoIntk4= +github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4/go.mod h1:o0O8gFiTfVp4g5QcQJ1iMLw6ROiy9BITaiBbEiwz9h8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= @@ -111,6 +113,8 @@ github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/go-echarts/go-echarts/v2 v2.4.2 h1:1FC3tGzsLSgdeO4Ltc3OAtcIiRomfEKxKX9oocIL68g= +github.com/go-echarts/go-echarts/v2 v2.4.2/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= diff --git a/internal/ledger.go b/internal/ledger.go index c096b935a..bd2f14847 100644 --- a/internal/ledger.go +++ b/internal/ledger.go @@ -2,10 +2,12 @@ package ledger import ( "fmt" + . "github.com/formancehq/go-libs/collectionutils" "github.com/formancehq/go-libs/time" "github.com/uptrace/bun" "regexp" "slices" + "strings" "github.com/formancehq/go-libs/metadata" ) @@ -145,6 +147,27 @@ func (f FeatureSet) With(feature, value string) FeatureSet { return ret } +func (f FeatureSet) String() string { + if len(f) == 0 { + return "" + } + keys := Keys(f) + slices.Sort(keys) + + ret := "" + for _, key := range keys { + ret = ret + "," + shortenFeature(key) + "=" + f[key] + } + + return ret[1:] +} + +func shortenFeature(feature string) string { + return strings.Join(Map(strings.Split(feature, "_"), func(from string) string { + return from[:1] + }), "") +} + type Configuration struct { Bucket string `json:"bucket" bun:"bucket,type:varchar(255)"` Metadata metadata.Metadata `json:"metadata" bun:"metadata,type:jsonb"` diff --git a/test/performance/README.md b/test/performance/README.md index 933681217..27ce4cd2a 100644 --- a/test/performance/README.md +++ b/test/performance/README.md @@ -29,72 +29,29 @@ go test -bench=. -run ^$ -tags it \ The output is a standard go bench output. -Example: +Additionally, you can pass the flag `-report.dir` to export results: +```shell +go test -bench=Write/testserver -run ^$ -tags it -report.dir . ``` -goos: darwin -goarch: arm64 -pkg: github.com/formancehq/ledger/test/performance -BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 84 15693529 ns/op 113.0 ms/transaction 62.81 t/s -BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 90 12960446 ns/op 100.0 ms/transaction 71.65 t/s -BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 87 13522271 ns/op 103.0 ms/transaction 70.18 t/s -BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 92 13460427 ns/op 102.0 ms/transaction 71.03 t/s -BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 66 15225778 ns/op 111.0 ms/transaction 59.71 t/s -BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 80 16243343 ns/op 119.0 ms/transaction 60.42 t/s -BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 55 19083905 ns/op 140.0 ms/transaction 48.15 t/s -BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 57 22134052 ns/op 155.0 ms/transaction 43.34 t/s -BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 75 15552648 ns/op 120.0 ms/transaction 58.70 t/s -BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 68 17866844 ns/op 134.0 ms/transaction 51.82 t/s -BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 81 13593212 ns/op 107.0 ms/transaction 66.39 t/s -BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 84 13560348 ns/op 104.0 ms/transaction 67.69 t/s -BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 86 14518456 ns/op 108.0 ms/transaction 67.05 t/s -BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 90 13041432 ns/op 101.0 ms/transaction 71.27 t/s -BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 76 14276990 ns/op 108.0 ms/transaction 66.02 t/s -BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 88 13430946 ns/op 103.0 ms/transaction 69.66 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8 72 15435962 ns/op 120.0 ms/transaction 58.49 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8 84 15247506 ns/op 114.0 ms/transaction 62.61 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8 78 16336328 ns/op 122.0 ms/transaction 59.34 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8 62 17001069 ns/op 125.0 ms/transaction 54.60 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8 62 18296797 ns/op 137.0 ms/transaction 51.63 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8 78 15578895 ns/op 118.0 ms/transaction 60.62 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8 57 18337382 ns/op 146.0 ms/transaction 45.01 t/s -BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8 66 17246153 ns/op 133.0 ms/transaction 52.94 t/s -``` +> [!WARNING] +> Benchmarks can be run in different environments: +> * core: We use the core only, no API. +> * testserver: A full test server is starter +> * remote: Target a remote ledger -There is the format: -``` -BenchmarkWrite/////- +The exported file is a csv. +You can use the [provided plot script](./plot/features_comparison.gp) to generate a bar chart for tps: +```shell +gnuplot -c plot/features_comparison.gp ``` -Where parameters are : -* env: - * core: the test use directly the core service of the ledger - * testserver: the test use a full running server - * remote : the test use a remote api -* transaction-type (see [scripts](./write_test.go)) - * world->bank : A transaction from `@world` to `@bank` - * world->any : A transaction from `@world` to `@dst:` - * any(unbounded)->any : A transaction from `@src:` to `@dst:` -* features: features enabled on the server (affect only servers supporting the feature) - * All combination of ledger features are tested. So the field `` will contains the list of the feature. For better visibility, features names are abbreviated. There is the code : - * HL => HASH_LOG : Possible configuration are `DISABLED` or `SYNC` - * PCEV => POST_COMMIT_EFFECTIVE_VOLUMES : Possible configurations are `DISABLED` or `SYNC` - * PCV => POST_COMMIT_VOLUMES : Possible configurations are `DISABLED` or `SYNC` - * IAS => INDEX_ADDRESS_SEGMENTS : Index individual segments address of accounts. Possible configurations are 'ON' or 'OFF'. - * AMH => ACCOUNT_METADATA_HISTORY : Historize accounts metadata - * TMH => TRANSACTION_METADATA_HISTORY : Historize transactions metadata +Each feature is tested against a test script involving a transaction from a source to a destination. +The benchmarks also test the minimal set of features and the full set of features. -If the flag `--include-ledger-in-results` is specified, the line will contain two additional arguments : -* bucket: bucket name where the ledger is created -* ledger: ledger name - -Example: -``` -BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC/5372990c/363e57d5-8 90 13808911 ns/op 104.0 ms/transaction 68.89 t/s -... -``` +Refer to [features](../../CONTRIBUTING.md/#features) for more information about features. -Other columns of the benchmark output are the metrics. -In addition to the standard metrics, we have two specific metrics : -* average transaction duration -* tps : transactions per second \ No newline at end of file +Three types of script are actually tested: +* world->bank : A transaction from `@world` to `@bank` +* world->any : A transaction from `@world` to `@dst:` +* any(unbounded)->any : A transaction from `@src:` to `@dst:` \ No newline at end of file diff --git a/test/performance/benchmark_test.go b/test/performance/benchmark_test.go index 701560b57..86fd170d0 100644 --- a/test/performance/benchmark_test.go +++ b/test/performance/benchmark_test.go @@ -9,7 +9,7 @@ import ( "sync/atomic" "testing" - "github.com/formancehq/go-libs/collectionutils" + . "github.com/formancehq/go-libs/collectionutils" "github.com/formancehq/go-libs/time" ledger "github.com/formancehq/ledger/internal" "github.com/google/uuid" @@ -24,23 +24,23 @@ type Benchmark struct { b *testing.B } -func (benchmark *Benchmark) Run(ctx context.Context) []*Report { - reports := make([]*Report, 0) +func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Report { + reports := make(map[string][]Report, 0) for envName, envFactory := range benchmark.EnvFactories { - scriptsKeys := collectionutils.Keys(benchmark.Scripts) + scriptsKeys := Keys(benchmark.Scripts) sort.Strings(scriptsKeys) for _, scriptName := range scriptsKeys { - for _, features := range buildAllPossibleConfigurations() { + for _, configuration := range buildAllPossibleConfigurations() { - testName := fmt.Sprintf("%s/%s/%s", envName, scriptName, features) + testName := fmt.Sprintf("%s/%s/%s", envName, scriptName, configuration) ledgerConfiguration := ledger.Configuration{ - Features: features, + Features: configuration.FeatureSet, Bucket: uuid.NewString()[:8], } ledgerConfiguration.SetDefaults() - report := newReport(ledgerConfiguration.Features, scriptName) + report := newReport(configuration, scriptName) benchmark.b.Run(testName, func(b *testing.B) { report.reset() @@ -81,7 +81,7 @@ func (benchmark *Benchmark) Run(ctx context.Context) []*Report { }) if report.TransactionsCount > 0 { - reports = append(reports, report) + reports[scriptName] = append(reports[scriptName], report) } } } diff --git a/test/performance/features_test.go b/test/performance/features_test.go index 64741c63e..cbece103f 100644 --- a/test/performance/features_test.go +++ b/test/performance/features_test.go @@ -3,64 +3,42 @@ package performance_test import ( - "slices" - "strings" - . "github.com/formancehq/go-libs/collectionutils" ledger "github.com/formancehq/ledger/internal" + "sort" ) -func buildAllPossibleConfigurations() []FeatureConfiguration { - possibleConfigurations := make([]FeatureConfiguration, 0) - - for _, feature := range Keys(ledger.FeatureConfigurations) { - optionsForFeature := Map(ledger.FeatureConfigurations[feature], func(from string) FeatureConfiguration { - return FeatureConfiguration{ - feature: from, - } +func buildAllPossibleConfigurations() []configuration { + possibleConfigurations := make([]configuration, 0) + possibleConfigurations = append(possibleConfigurations, configuration{ + Name: "MINIMAL", + FeatureSet: ledger.MinimalFeatureSet, + }) + + fullConfiguration := ledger.MinimalFeatureSet + features := Keys(ledger.FeatureConfigurations) + sort.Strings(features) + + for _, feature := range features { + possibleConfigurations = append(possibleConfigurations, configuration{ + Name: feature, + FeatureSet: ledger.MinimalFeatureSet.With(feature, ledger.FeatureConfigurations[feature][0]), }) - if len(possibleConfigurations) == 0 { - possibleConfigurations = append(possibleConfigurations, optionsForFeature...) - } else { - newPossibleConfigurations := make([]FeatureConfiguration, 0) - for _, existing := range possibleConfigurations { - for _, optionForFeature := range optionsForFeature { - for k, v := range optionForFeature { - replaced := make(map[string]string) - for k, v := range existing { - replaced[k] = v - } - replaced[k] = v - newPossibleConfigurations = append(newPossibleConfigurations, replaced) - } - } - } - possibleConfigurations = newPossibleConfigurations - } + fullConfiguration = fullConfiguration.With(feature, ledger.FeatureConfigurations[feature][0]) } + possibleConfigurations = append(possibleConfigurations, configuration{ + Name: "FULL", + FeatureSet: fullConfiguration, + }) return possibleConfigurations } -type FeatureConfiguration map[string]string - -func (cfg FeatureConfiguration) String() string { - if len(cfg) == 0 { - return "" - } - keys := Keys(cfg) - slices.Sort(keys) - - ret := "" - for _, key := range keys { - ret = ret + "," + shortenFeature(key) + "=" + cfg[key] - } - - return ret[1:] +type configuration struct { + Name string + FeatureSet ledger.FeatureSet } -func shortenFeature(feature string) string { - return strings.Join(Map(strings.Split(feature, "_"), func(from string) string { - return from[:1] - }), "") +func (c configuration) String() string { + return c.Name } diff --git a/test/performance/main_test.go b/test/performance/main_test.go index 8b5bccc9f..9172e6376 100644 --- a/test/performance/main_test.go +++ b/test/performance/main_test.go @@ -32,7 +32,8 @@ var ( ledgerURL string parallelism int64 - reportFile string + reportDir string + testCore bool envFactories = make(map[string]EnvFactory) ) @@ -43,16 +44,27 @@ func init() { flag.StringVar(&authClientSecret, "client.secret", "", "Client secret") flag.StringVar(&ledgerURL, "ledger.url", "", "Ledger url") flag.StringVar(&authIssuerURL, "auth.url", "", "Auth url (ignored if --stack.url is specified)") - flag.StringVar(&reportFile, "report.file", "", "Location to write report file") + flag.StringVar(&reportDir, "report.dir", "", "Location to write report files") flag.Int64Var(¶llelism, "parallelism", 1, "Parallelism (default 1). Values is multiplied by GOMAXPROCS") + flag.BoolVar(&testCore, "test-core", false, "Test core only") } func TestMain(m *testing.M) { flag.Parse() utils.WithTestMain(func(t *utils.TestingTForMain) int { - if stackURL != "" && ledgerURL != "" { - t.Errorf("Cannot specify both --stack.url and --ledger.url") + selectedEnv := 0 + if stackURL != "" { + selectedEnv++ + } + if ledgerURL != "" { + selectedEnv++ + } + if testCore { + selectedEnv++ + } + if selectedEnv > 1 { + t.Errorf("Cannot specify both --stack.url, --ledger.url or --test-core") t.FailNow() } @@ -61,6 +73,8 @@ func TestMain(m *testing.M) { setupRemoteStackEnv() case ledgerURL != "": setRemoteLedgerEnv() + case testCore: + setCoreEnv() default: setupLocalEnv(t) } @@ -78,6 +92,9 @@ func setupLocalEnv(t *utils.TestingTForMain) { dockerPool, pgtesting.WithPGCrypto(), ) + envFactories = map[string]EnvFactory{ + "testserver": NewTestServerEnvFactory(pgServer), + } } // setupRemoveEnv configure a remote env @@ -90,6 +107,10 @@ func setRemoteLedgerEnv() { envFactories["remote"] = NewRemoteLedgerEnvFactory(getHttpClient(authIssuerURL), ledgerURL) } +func setCoreEnv() { + envFactories["core"] = NewCoreEnvFactory(pgServer) +} + func getHttpClient(authUrl string) *http.Client { httpClient := &http.Client{ Transport: &http.Transport{ diff --git a/test/performance/plot/features_comparison.gp b/test/performance/plot/features_comparison.gp new file mode 100644 index 000000000..71677c7cd --- /dev/null +++ b/test/performance/plot/features_comparison.gp @@ -0,0 +1,12 @@ + +set terminal cgm width 10/2.54*72 +set term png + +set output "features_comparison.png" +set boxwidth 0.9 relative +set style data histograms +set style histogram cluster +set style fill solid 1.0 border lt -1 +set xtics rotate by 90 right + +plot for [COL=2:5] 'features_comparison.csv' using COL:xticlabels(1) title col diff --git a/test/performance/report_test.go b/test/performance/report_test.go index 250437653..b949d7745 100644 --- a/test/performance/report_test.go +++ b/test/performance/report_test.go @@ -3,7 +3,6 @@ package performance_test import ( - "encoding/json" "sync" "sync/atomic" @@ -13,7 +12,6 @@ import ( type Report struct { mu *sync.Mutex - Features map[string]string Start time.Time End time.Time @@ -21,17 +19,7 @@ type Report struct { TotalLatency *atomic.Int64 TransactionsCount int Name string -} - -func (r *Report) MarshalJSON() ([]byte, error) { - type Aux Report - return json.Marshal(struct { - Aux - TotalLatency int64 - }{ - Aux: Aux(*r), - TotalLatency: r.TotalLatency.Load(), - }) + Configuration configuration } func (r *Report) TPS() float64 { @@ -56,11 +44,11 @@ func (r *Report) reset() { r.Start = time.Now() } -func newReport(features map[string]string, name string) *Report { - ret := &Report{ - Name: name, - Features: features, - mu: &sync.Mutex{}, +func newReport(configuration configuration, name string) Report { + ret := Report{ + Name: name, + Configuration: configuration, + mu: &sync.Mutex{}, } ret.reset() return ret diff --git a/test/performance/write_test.go b/test/performance/write_test.go index b3178c703..93e5437fa 100644 --- a/test/performance/write_test.go +++ b/test/performance/write_test.go @@ -3,20 +3,22 @@ package performance_test import ( - "encoding/json" + "encoding/csv" "fmt" + . "github.com/formancehq/go-libs/collectionutils" + "github.com/formancehq/go-libs/logging" "github.com/stretchr/testify/require" "os" "path/filepath" + "sort" "testing" - - "github.com/formancehq/go-libs/logging" ) var scripts = map[string]func(int) (string, map[string]string){ "world->bank": worldToBank, "world->any": worldToAny, "any(unbounded)->any": anyUnboundedToAny, + "any(bounded)->any": anyBoundedToAny, } func worldToBank(_ int) (string, map[string]string) { @@ -55,28 +57,55 @@ send [USD/2 100] ( } } -func BenchmarkWrite(b *testing.B) { - - // Set default env factories if not defined (remote mode not used) - if len(envFactories) == 0 { - envFactories = map[string]EnvFactory{ - "core": NewCoreEnvFactory(pgServer), - "testserver": NewTestServerEnvFactory(pgServer), +func anyBoundedToAny(id int) (string, map[string]string) { + return fmt.Sprintf(` +vars { + account $source + account $destination +} +send [USD/2 100] ( + source = $source allowing overdraft up to [USD/2 %d] + destination = $destination +)`, (id+1)*100), map[string]string{ + "source": fmt.Sprintf("src:%d", id), + "destination": fmt.Sprintf("dst:%d", id), } - } +} + +// todo: add response time +func BenchmarkWrite(b *testing.B) { // Execute benchmarks reports := New(b, envFactories, scripts).Run(logging.TestingContext()) // Write report - if reportFile != "" { - require.NoError(b, os.MkdirAll(filepath.Dir(reportFile), 0755)) + if reportDir != "" { + require.NoError(b, os.MkdirAll(reportDir, 0755)) - f, err := os.Create(reportFile) + featureComparisonCSVLocation := filepath.Join(reportDir, "features_comparison.csv") + featureComparisonCSVFile, err := os.Create(featureComparisonCSVLocation) require.NoError(b, err) - enc := json.NewEncoder(f) - enc.SetIndent("", " ") - require.NoError(b, enc.Encode(reports)) + w := csv.NewWriter(featureComparisonCSVFile) + w.Comma = ' ' + + scripts := Keys(reports) + sort.Strings(scripts) + + csvLine := make([]string, 0) + csvLine = append(csvLine, "scenario") + csvLine = append(csvLine, scripts...) + require.NoError(b, w.Write(csvLine)) + + for line := 0 ; line < len(reports[scripts[0]]) ; line++ { + csvLine = make([]string, 0) + csvLine = append(csvLine, reports[scripts[0]][line].Configuration.Name) + for j := 0 ; j < len(scripts) ; j++ { + csvLine = append(csvLine, fmt.Sprint(reports[scripts[j]][line].TPS())) + } + require.NoError(b, w.Write(csvLine)) + } + + w.Flush() } -} +} \ No newline at end of file