feat: refine benchmark and add plot script

formancehq · Oct 16, 2024 · 8937067 · 8937067
1 parent a5188ad
commit 8937067
Show file tree

Hide file tree

Showing 12 changed files with 176 additions and 161 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 cover.out
 go.work*
 *.jar
+report
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -121,7 +121,7 @@ Additionally, each ledger is created on a bucket, `quickstart` is installed on b
 >
 > But, automatically created ledgers by v1 api will create a new bucket with the same name as the ledger. That's for compatibility reasons regarding ledger v1 behavior.
 
-***Buckets***
+### Buckets
 
 To create a ledger on a specific bucket, use the command:
 
@@ -150,7 +150,7 @@ $ curl http://localhost:3068/v2/testing | jq
 
 Under the hood, a bucket is a Postgres schema. You can use the bucket feature to implement some kind of horizontal scaling.
 
-***Features***
+### Features
 
 Each usage of the ledger service, is different.
 Some usage involve a high write throughput, some other involve high read throughput, custom aggregation etc...

diff --git a/go.mod b/go.mod
@@ -46,6 +46,7 @@ require (
 require (
 	dario.cat/mergo v1.0.1 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4 // indirect
 	github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
 	github.com/IBM/sarama v1.43.3 // indirect
 	github.com/Microsoft/go-winio v0.6.2 // indirect
@@ -86,6 +87,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
 	github.com/go-chi/render v1.0.3 // indirect
+	github.com/go-echarts/go-echarts/v2 v2.4.2 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect

diff --git a/go.sum b/go.sum
@@ -2,6 +2,8 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
 dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4 h1:tTl76UtV0KnHdcIxHAhq31NNO/B/r0+LnPsvUoIntk4=
+github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4/go.mod h1:o0O8gFiTfVp4g5QcQJ1iMLw6ROiy9BITaiBbEiwz9h8=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
 github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA=
@@ -111,6 +113,8 @@ github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
 github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
 github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4=
 github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0=
+github.com/go-echarts/go-echarts/v2 v2.4.2 h1:1FC3tGzsLSgdeO4Ltc3OAtcIiRomfEKxKX9oocIL68g=
+github.com/go-echarts/go-echarts/v2 v2.4.2/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
 github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=

diff --git a/internal/ledger.go b/internal/ledger.go
@@ -2,10 +2,12 @@ package ledger
 
 import (
 	"fmt"
+	. "github.com/formancehq/go-libs/collectionutils"
 	"github.com/formancehq/go-libs/time"
 	"github.com/uptrace/bun"
 	"regexp"
 	"slices"
+	"strings"
 
 	"github.com/formancehq/go-libs/metadata"
 )
@@ -145,6 +147,27 @@ func (f FeatureSet) With(feature, value string) FeatureSet {
 	return ret
 }
 
+func (f FeatureSet) String() string {
+	if len(f) == 0 {
+		return ""
+	}
+	keys := Keys(f)
+	slices.Sort(keys)
+
+	ret := ""
+	for _, key := range keys {
+		ret = ret + "," + shortenFeature(key) + "=" + f[key]
+	}
+
+	return ret[1:]
+}
+
+func shortenFeature(feature string) string {
+	return strings.Join(Map(strings.Split(feature, "_"), func(from string) string {
+		return from[:1]
+	}), "")
+}
+
 type Configuration struct {
 	Bucket   string            `json:"bucket" bun:"bucket,type:varchar(255)"`
 	Metadata metadata.Metadata `json:"metadata" bun:"metadata,type:jsonb"`

diff --git a/test/performance/README.md b/test/performance/README.md
@@ -29,72 +29,29 @@ go test -bench=. -run ^$ -tags it \
 
 The output is a standard go bench output.
 
-Example: 
+Additionally, you can pass the flag `-report.dir` to export results:
+```shell
+go test -bench=Write/testserver -run ^$ -tags it -report.dir .
 ```
-goos: darwin
-goarch: arm64
-pkg: github.com/formancehq/ledger/test/performance
-BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC-8                      84          15693529 ns/op               113.0 ms/transaction             62.81 t/s
-BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8                  90          12960446 ns/op               100.0 ms/transaction             71.65 t/s
-BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8                  87          13522271 ns/op               103.0 ms/transaction             70.18 t/s
-BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8              92          13460427 ns/op               102.0 ms/transaction             71.03 t/s
-BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8                  66          15225778 ns/op               111.0 ms/transaction             59.71 t/s
-BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8              80          16243343 ns/op               119.0 ms/transaction             60.42 t/s
-BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8              55          19083905 ns/op               140.0 ms/transaction             48.15 t/s
-BenchmarkWrite/remote/world->bank/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8                  57          22134052 ns/op               155.0 ms/transaction             43.34 t/s
-BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8                               75          15552648 ns/op               120.0 ms/transaction             58.70 t/s
-BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8                           68          17866844 ns/op               134.0 ms/transaction             51.82 t/s
-BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8                           81          13593212 ns/op               107.0 ms/transaction             66.39 t/s
-BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8                       84          13560348 ns/op               104.0 ms/transaction             67.69 t/s
-BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8                           86          14518456 ns/op               108.0 ms/transaction             67.05 t/s
-BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8                       90          13041432 ns/op               101.0 ms/transaction             71.27 t/s
-BenchmarkWrite/remote/world->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8                       76          14276990 ns/op               108.0 ms/transaction             66.02 t/s
-BenchmarkWrite/remote/world->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8                   88          13430946 ns/op               103.0 ms/transaction             69.66 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=SYNC-8                      72          15435962 ns/op               120.0 ms/transaction             58.49 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=SYNC-8                  84          15247506 ns/op               114.0 ms/transaction             62.61 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=SYNC-8                  78          16336328 ns/op               122.0 ms/transaction             59.34 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=SYNC-8              62          17001069 ns/op               125.0 ms/transaction             54.60 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=SYNC,PCV=DISABLED-8                  62          18296797 ns/op               137.0 ms/transaction             51.63 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=SYNC,PCV=DISABLED-8              78          15578895 ns/op               118.0 ms/transaction             60.62 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=SYNC,PCEV=DISABLED,PCV=DISABLED-8              57          18337382 ns/op               146.0 ms/transaction             45.01 t/s
-BenchmarkWrite/remote/any(unbounded)->any/HL=DISABLED,PCEV=DISABLED,PCV=DISABLED-8          66          17246153 ns/op               133.0 ms/transaction             52.94 t/s
 
-```
+> [!WARNING]
+> Benchmarks can be run in different environments:
+> * core: We use the core only, no API.
+> * testserver: A full test server is starter
+> * remote: Target a remote ledger
 
-There is the format:
-```
-BenchmarkWrite/<env>/<transaction-type>/<features>/<bucket>/<ledger>-<cpu count>                      <iteration count>          <ns by op>               <average transaction duration>             <tps>
+The exported file is a csv. 
+You can use the [provided plot script](./plot/features_comparison.gp) to generate a bar chart for tps:
+```shell
+gnuplot -c plot/features_comparison.gp
 ```
 
-Where parameters are :
-* env: 
-  * core: the test use directly the core service of the ledger
-  * testserver: the test use a full running server
-  * remote : the test use a remote api
-* transaction-type (see [scripts](./write_test.go))
-  * world->bank : A transaction from `@world` to `@bank`
-  * world->any : A transaction from `@world` to `@dst:<iteration>`
-  * any(unbounded)->any : A transaction from `@src:<iteration>` to `@dst:<iteration>`
-* features: features enabled on the server (affect only servers supporting the feature)
-  * All combination of ledger features are tested. So the field `<features>` will contains the list of the feature. For better visibility, features names are abbreviated. There is the code : 
-    * HL => HASH_LOG : Possible configuration are `DISABLED` or `SYNC`
-    * PCEV => POST_COMMIT_EFFECTIVE_VOLUMES : Possible configurations are `DISABLED` or `SYNC`
-    * PCV => POST_COMMIT_VOLUMES : Possible configurations are `DISABLED` or `SYNC`
-    * IAS => INDEX_ADDRESS_SEGMENTS : Index individual segments address of accounts. Possible configurations are 'ON' or 'OFF'.
-    * AMH => ACCOUNT_METADATA_HISTORY : Historize accounts metadata
-    * TMH => TRANSACTION_METADATA_HISTORY : Historize transactions metadata
+Each feature is tested against a test script involving a transaction from a source to a destination.
+The benchmarks also test the minimal set of features and the full set of features.
 
-If the flag `--include-ledger-in-results` is specified, the line will contain two additional arguments :
-* bucket: bucket name where the ledger is created
-* ledger: ledger name
-
-Example:
-```
-BenchmarkWrite/remote/world->bank/HL=SYNC,PCEV=SYNC,PCV=SYNC/5372990c/363e57d5-8                      90          13808911 ns/op               104.0 ms/transaction             68.89 t/s 
-...
-```
+Refer to [features](../../CONTRIBUTING.md/#features) for more information about features.
 
-Other columns of the benchmark output are the metrics.
-In addition to the standard metrics, we have two specific metrics : 
-* average transaction duration
-* tps : transactions per second
+Three types of script are actually tested:
+* world->bank : A transaction from `@world` to `@bank`
+* world->any : A transaction from `@world` to `@dst:<iteration>`
+* any(unbounded)->any : A transaction from `@src:<iteration>` to `@dst:<iteration>`
diff --git a/test/performance/benchmark_test.go b/test/performance/benchmark_test.go
@@ -9,7 +9,7 @@ import (
 	"sync/atomic"
 	"testing"
 
-	"github.com/formancehq/go-libs/collectionutils"
+	. "github.com/formancehq/go-libs/collectionutils"
 	"github.com/formancehq/go-libs/time"
 	ledger "github.com/formancehq/ledger/internal"
 	"github.com/google/uuid"
@@ -24,23 +24,23 @@ type Benchmark struct {
 	b       *testing.B
 }
 
-func (benchmark *Benchmark) Run(ctx context.Context) []*Report {
-	reports := make([]*Report, 0)
+func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Report {
+	reports := make(map[string][]Report, 0)
 	for envName, envFactory := range benchmark.EnvFactories {
-		scriptsKeys := collectionutils.Keys(benchmark.Scripts)
+		scriptsKeys := Keys(benchmark.Scripts)
 		sort.Strings(scriptsKeys)
 
 		for _, scriptName := range scriptsKeys {
-			for _, features := range buildAllPossibleConfigurations() {
+			for _, configuration := range buildAllPossibleConfigurations() {
 
-				testName := fmt.Sprintf("%s/%s/%s", envName, scriptName, features)
+				testName := fmt.Sprintf("%s/%s/%s", envName, scriptName, configuration)
 
 				ledgerConfiguration := ledger.Configuration{
-					Features: features,
+					Features: configuration.FeatureSet,
 					Bucket:   uuid.NewString()[:8],
 				}
 				ledgerConfiguration.SetDefaults()
-				report := newReport(ledgerConfiguration.Features, scriptName)
+				report := newReport(configuration, scriptName)
 
 				benchmark.b.Run(testName, func(b *testing.B) {
 					report.reset()
@@ -81,7 +81,7 @@ func (benchmark *Benchmark) Run(ctx context.Context) []*Report {
 				})
 
 				if report.TransactionsCount > 0 {
-					reports = append(reports, report)
+					reports[scriptName] = append(reports[scriptName], report)
 				}
 			}
 		}

diff --git a/test/performance/features_test.go b/test/performance/features_test.go
@@ -3,64 +3,42 @@
 package performance_test
 
 import (
-	"slices"
-	"strings"
-
 	. "github.com/formancehq/go-libs/collectionutils"
 	ledger "github.com/formancehq/ledger/internal"
+	"sort"
 )
 
-func buildAllPossibleConfigurations() []FeatureConfiguration {
-	possibleConfigurations := make([]FeatureConfiguration, 0)
-
-	for _, feature := range Keys(ledger.FeatureConfigurations) {
-		optionsForFeature := Map(ledger.FeatureConfigurations[feature], func(from string) FeatureConfiguration {
-			return FeatureConfiguration{
-				feature: from,
-			}
+func buildAllPossibleConfigurations() []configuration {
+	possibleConfigurations := make([]configuration, 0)
+	possibleConfigurations = append(possibleConfigurations, configuration{
+		Name:       "MINIMAL",
+		FeatureSet: ledger.MinimalFeatureSet,
+	})
+
+	fullConfiguration := ledger.MinimalFeatureSet
+	features := Keys(ledger.FeatureConfigurations)
+	sort.Strings(features)
+
+	for _, feature := range features {
+		possibleConfigurations = append(possibleConfigurations, configuration{
+			Name:       feature,
+			FeatureSet: ledger.MinimalFeatureSet.With(feature, ledger.FeatureConfigurations[feature][0]),
 		})
-		if len(possibleConfigurations) == 0 {
-			possibleConfigurations = append(possibleConfigurations, optionsForFeature...)
-		} else {
-			newPossibleConfigurations := make([]FeatureConfiguration, 0)
-			for _, existing := range possibleConfigurations {
-				for _, optionForFeature := range optionsForFeature {
-					for k, v := range optionForFeature {
-						replaced := make(map[string]string)
-						for k, v := range existing {
-							replaced[k] = v
-						}
-						replaced[k] = v
-						newPossibleConfigurations = append(newPossibleConfigurations, replaced)
-					}
-				}
-			}
-			possibleConfigurations = newPossibleConfigurations
-		}
+		fullConfiguration = fullConfiguration.With(feature, ledger.FeatureConfigurations[feature][0])
 	}
+	possibleConfigurations = append(possibleConfigurations, configuration{
+		Name:       "FULL",
+		FeatureSet: fullConfiguration,
+	})
 
 	return possibleConfigurations
 }
 
-type FeatureConfiguration map[string]string
-
-func (cfg FeatureConfiguration) String() string {
-	if len(cfg) == 0 {
-		return ""
-	}
-	keys := Keys(cfg)
-	slices.Sort(keys)
-
-	ret := ""
-	for _, key := range keys {
-		ret = ret + "," + shortenFeature(key) + "=" + cfg[key]
-	}
-
-	return ret[1:]
+type configuration struct {
+	Name string
+	FeatureSet ledger.FeatureSet
 }
 
-func shortenFeature(feature string) string {
-	return strings.Join(Map(strings.Split(feature, "_"), func(from string) string {
-		return from[:1]
-	}), "")
+func (c configuration) String() string {
+	return c.Name
 }