feat: refine benchmarks

formancehq · Oct 23, 2024 · 39c609e · 39c609e
1 parent 50468a3
commit 39c609e
Show file tree

Hide file tree

Showing 9 changed files with 186 additions and 248 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 cover.out
 go.work*
 *.jar
-report
+report
+node_modules
diff --git a/test/performance/README.md b/test/performance/README.md
@@ -31,20 +31,10 @@ The output is a standard go bench output.
 
 Additionally, you can pass the flag `-report.dir` to export results:
 ```shell
-go test -bench=Write/testserver -run ^$ -tags it -report.dir .
+go test -bench=testserver -run ^$ -tags it -report.dir .
 ```
 
-> [!WARNING]
-> Benchmarks can be run in different environments:
-> * core: We use the core only, no API.
-> * testserver: A full test server is starter
-> * remote: Target a remote ledger
-
-The exported file is a csv. 
-You can use the [provided plot script](./plot/features_comparison.gp) to generate a bar chart for tps:
-```shell
-gnuplot -c plot/features_comparison_tps.gp
-```
+The exported files are PNG. 
 
 Each feature is tested against a test script involving a transaction from a source to a destination.
 The benchmarks also test the minimal set of features and the full set of features.

diff --git a/test/performance/benchmark_test.go b/test/performance/benchmark_test.go
@@ -17,84 +17,82 @@ import (
 )
 
 type Benchmark struct {
-	EnvFactories map[string]EnvFactory
-	Scripts      map[string]func(int) (string, map[string]string)
+	EnvFactory EnvFactory
+	Scripts    map[string]func(int) (string, map[string]string)
 
 	reports map[string]map[string]*Report
 	b       *testing.B
 }
 
 func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Report {
 	reports := make(map[string][]Report, 0)
-	for envName, envFactory := range benchmark.EnvFactories {
-		scriptsKeys := Keys(benchmark.Scripts)
-		sort.Strings(scriptsKeys)
+	scriptsKeys := Keys(benchmark.Scripts)
+	sort.Strings(scriptsKeys)
 
-		for _, scriptName := range scriptsKeys {
-			for _, configuration := range buildAllPossibleConfigurations() {
+	for _, scriptName := range scriptsKeys {
+		for _, configuration := range buildAllPossibleConfigurations() {
 
-				testName := fmt.Sprintf("%s/%s/%s", envName, scriptName, configuration)
+			testName := fmt.Sprintf("%s/%s", scriptName, configuration)
 
-				ledgerConfiguration := ledger.Configuration{
-					Features: configuration.FeatureSet,
-					Bucket:   uuid.NewString()[:8],
+			ledgerConfiguration := ledger.Configuration{
+				Features: configuration.FeatureSet,
+				Bucket:   uuid.NewString()[:8],
+			}
+			ledgerConfiguration.SetDefaults()
+			report := newReport(configuration, scriptName)
+
+			benchmark.b.Run(testName, func(b *testing.B) {
+				report.reset()
+				l := ledger.Ledger{
+					Configuration: ledgerConfiguration,
+					Name:          uuid.NewString()[:8],
 				}
-				ledgerConfiguration.SetDefaults()
-				report := newReport(configuration, scriptName)
-
-				benchmark.b.Run(testName, func(b *testing.B) {
-					report.reset()
-					l := ledger.Ledger{
-						Configuration: ledgerConfiguration,
-						Name:          uuid.NewString()[:8],
-					}
 
-					cpt := atomic.Int64{}
+				cpt := atomic.Int64{}
 
-					env := envFactory.Create(ctx, b, l)
-					b.Logf("ledger: %s/%s", l.Bucket, l.Name)
+				env := envFactory.Create(ctx, b, l)
+				b.Logf("ledger: %s/%s", l.Bucket, l.Name)
 
-					b.SetParallelism(int(parallelism))
-					b.ResetTimer()
-					b.RunParallel(func(pb *testing.PB) {
-						for pb.Next() {
-							iteration := int(cpt.Add(1))
+				b.SetParallelism(int(parallelism))
+				b.ResetTimer()
+				b.RunParallel(func(pb *testing.PB) {
+					for pb.Next() {
+						iteration := int(cpt.Add(1))
 
-							script, vars := benchmark.Scripts[scriptName](iteration)
-							now := time.Now()
-							_, err := env.Executor().ExecuteScript(ctx, script, vars)
-							require.NoError(b, err)
+						script, vars := benchmark.Scripts[scriptName](iteration)
+						now := time.Now()
+						_, err := env.Executor().ExecuteScript(ctx, script, vars)
+						require.NoError(b, err)
 
-							report.registerTransactionLatency(time.Since(now))
-						}
-					})
-					b.StopTimer()
-					report.End = time.Now()
+						report.registerTransactionLatency(time.Since(now))
+					}
+				})
+				b.StopTimer()
+				report.End = time.Now()
 
-					b.ReportMetric(report.TPS(), "t/s")
-					b.ReportMetric(float64(report.AverageDuration().Milliseconds()), "ms/transaction")
+				b.ReportMetric(report.TPS(), "t/s")
+				b.ReportMetric(float64(report.Tachymeter.Calc().Time.Avg.Milliseconds()), "ms/transaction")
 
-					stopContext, cancel := context.WithTimeout(ctx, 10*time.Second)
-					b.Cleanup(cancel)
+				stopContext, cancel := context.WithTimeout(ctx, 10*time.Second)
+				b.Cleanup(cancel)
 
-					require.NoError(benchmark.b, env.Stop(stopContext))
-				})
+				require.NoError(benchmark.b, env.Stop(stopContext))
+			})
 
-				if report.Tachymeter.Count > 0 {
-					reports[scriptName] = append(reports[scriptName], report)
-				}
+			if report.Tachymeter.Count > 0 {
+				reports[scriptName] = append(reports[scriptName], report)
 			}
 		}
 	}
 
 	return reports
 }
 
-func New(b *testing.B, envFactories map[string]EnvFactory, scripts map[string]func(int) (string, map[string]string)) *Benchmark {
+func New(b *testing.B, envFactory EnvFactory, scripts map[string]func(int) (string, map[string]string)) *Benchmark {
 	return &Benchmark{
-		b:            b,
-		EnvFactories: envFactories,
-		Scripts:      scripts,
-		reports:      make(map[string]map[string]*Report),
+		b:          b,
+		EnvFactory: envFactory,
+		Scripts:    scripts,
+		reports:    make(map[string]map[string]*Report),
 	}
 }
diff --git a/test/performance/env_remote_stack_test.go b/test/performance/env_remote_stack_test.go
diff --git a/test/performance/main_test.go b/test/performance/main_test.go
@@ -6,10 +6,10 @@ import (
 	"context"
 	"crypto/tls"
 	"flag"
+	"github.com/formancehq/go-libs/logging"
 	"net/http"
 	"testing"
 
-	"github.com/formancehq/go-libs/logging"
 	"github.com/formancehq/go-libs/testing/docker"
 	"github.com/formancehq/go-libs/testing/platform/pgtesting"
 	"github.com/formancehq/go-libs/testing/utils"
@@ -35,7 +35,7 @@ var (
 	reportDir   string
 	testCore bool
 
-	envFactories = make(map[string]EnvFactory)
+	envFactory EnvFactory
 )
 
 func init() {
@@ -46,7 +46,7 @@ func init() {
 	flag.StringVar(&authIssuerURL, "auth.url", "", "Auth url (ignored if --stack.url is specified)")
 	flag.StringVar(&reportDir, "report.dir", "", "Location to write report files")
 	flag.Int64Var(&parallelism, "parallelism", 1, "Parallelism (default 1). Values is multiplied by GOMAXPROCS")
-	flag.BoolVar(&testCore, "test-core", false, "Test core only")
+	flag.BoolVar(&testCore, "core", false, "Test core only")
 }
 
 func TestMain(m *testing.M) {
@@ -70,47 +70,27 @@ func TestMain(m *testing.M) {
 
 		switch {
 		case stackURL != "":
-			setupRemoteStackEnv()
+			envFactory = NewRemoteLedgerEnvFactory(getHttpClient(stackURL+"/api/auth"), stackURL+"/api/ledger")
 		case ledgerURL != "":
-			setRemoteLedgerEnv()
+			envFactory = NewRemoteLedgerEnvFactory(getHttpClient(authIssuerURL), ledgerURL)
 		case testCore:
-			setCoreEnv()
+			envFactory = NewCoreEnvFactory(pgServer)
 		default:
-			setupLocalEnv(t)
+			// Configure the environment to run benchmarks locally.
+			// Start a docker connection and create a new postgres server.
+			dockerPool = docker.NewPool(t, logging.Testing())
+			pgServer = pgtesting.CreatePostgresServer(
+				t,
+				dockerPool,
+				pgtesting.WithPGCrypto(),
+			)
+			envFactory = NewTestServerEnvFactory(pgServer)
 		}
 
 		return m.Run()
 	})
 }
 
-// setupLocalEnv configure the environment for running benchmarks locally
-// is it start a docker connection and create a new postgres server
-func setupLocalEnv(t *utils.TestingTForMain) {
-	dockerPool = docker.NewPool(t, logging.Testing())
-	pgServer = pgtesting.CreatePostgresServer(
-		t,
-		dockerPool,
-		pgtesting.WithPGCrypto(),
-	)
-	envFactories = map[string]EnvFactory{
-		"testserver": NewTestServerEnvFactory(pgServer),
-	}
-}
-
-// setupRemoveEnv configure a remote env
-func setupRemoteStackEnv() {
-	envFactories["remote"] = NewRemoteStackEnvFactory(getHttpClient(stackURL+"/api/auth"), stackURL)
-}
-
-// setupRemoveEnv configure a remote env
-func setRemoteLedgerEnv() {
-	envFactories["remote"] = NewRemoteLedgerEnvFactory(getHttpClient(authIssuerURL), ledgerURL)
-}
-
-func setCoreEnv() {
-	envFactories["core"] = NewCoreEnvFactory(pgServer)
-}
-
 func getHttpClient(authUrl string) *http.Client {
 	httpClient := &http.Client{
 		Transport: &http.Transport{

diff --git a/test/performance/plot/features_comparison_latency.gp b/test/performance/plot/features_comparison_latency.gp
@@ -0,0 +1,12 @@
+
+set terminal cgm width 10/2.54*72
+set term png
+
+set output "{{.OutputFile}}"
+set boxwidth 0.9 relative
+set style data histograms
+set style histogram rowstacked
+set style fill solid 1.0 border lt -1
+set xtics rotate by 90 right
+
+plot for [COL=2:3] '{{.Datafile}}' using COL:xticlabels(1) title col