feat: add report file on benchmarks

test/performance/benchmark_test.go

@@ -24,7 +24,8 @@ type Benchmark struct {
 	b       *testing.B
 }
 
-func (benchmark *Benchmark) Run(ctx context.Context) error {
+func (benchmark *Benchmark) Run(ctx context.Context) []*Report {
+	reports := make([]*Report, 0)
 	for envName, envFactory := range benchmark.EnvFactories {
 		scriptsKeys := collectionutils.Keys(benchmark.Scripts)
 		sort.Strings(scriptsKeys)
@@ -34,19 +35,21 @@ func (benchmark *Benchmark) Run(ctx context.Context) error {
 
 				testName := fmt.Sprintf("%s/%s/%s", envName, scriptName, features)
 
+				ledgerConfiguration := ledger.Configuration{
+					Features: features,
+					Bucket:   uuid.NewString()[:8],
+				}
+				ledgerConfiguration.SetDefaults()
+				report := newReport(ledgerConfiguration.Features, scriptName)
+
 				benchmark.b.Run(testName, func(b *testing.B) {
-					ledgerConfiguration := ledger.Configuration{
-						Features: features,
-						Bucket:   uuid.NewString()[:8],
-					}
-					ledgerConfiguration.SetDefaults()
+					report.reset()
 					l := ledger.Ledger{
 						Configuration: ledgerConfiguration,
 						Name:          uuid.NewString()[:8],
 					}
 
 					cpt := atomic.Int64{}
-					report := newReport(ledgerConfiguration.Features)
 
 					env := envFactory.Create(ctx, b, l)
 					b.Logf("ledger: %s/%s", l.Bucket, l.Name)
@@ -66,7 +69,7 @@ func (benchmark *Benchmark) Run(ctx context.Context) error {
 						}
 					})
 					b.StopTimer()
-					report.endOfBench = time.Now()
+					report.End = time.Now()
 
 					b.ReportMetric(report.TPS(), "t/s")
 					b.ReportMetric(float64(report.AverageDuration().Milliseconds()), "ms/transaction")
@@ -76,11 +79,15 @@ func (benchmark *Benchmark) Run(ctx context.Context) error {
 
 					require.NoError(benchmark.b, env.Stop(stopContext))
 				})
+
+				if report.TransactionsCount > 0 {
+					reports = append(reports, report)
+				}
 			}
 		}
 	}
 
-	return nil
+	return reports
 }
 
 func New(b *testing.B, envFactories map[string]EnvFactory, scripts map[string]func(int) (string, map[string]string)) *Benchmark {

test/performance/main_test.go

@@ -32,6 +32,7 @@ var (
 	ledgerURL     string
 
 	parallelism int64
+	reportFile string
 
 	envFactories = make(map[string]EnvFactory)
 )
@@ -42,6 +43,7 @@ func init() {
 	flag.StringVar(&authClientSecret, "client.secret", "", "Client secret")
 	flag.StringVar(&ledgerURL, "ledger.url", "", "Ledger url")
 	flag.StringVar(&authIssuerURL, "auth.url", "", "Auth url (ignored if --stack.url is specified)")
+	flag.StringVar(&reportFile, "report.file", "", "Location to write report file")
 	flag.Int64Var(&parallelism, "parallelism", 1, "Parallelism (default 1). Values is multiplied by GOMAXPROCS")
 }
 

test/performance/report_test.go

@@ -3,43 +3,65 @@
 package performance_test
 
 import (
+	"encoding/json"
 	"sync"
 	"sync/atomic"
 
 	"github.com/formancehq/go-libs/time"
 )
 
 type Report struct {
-	mu sync.Mutex
+	mu *sync.Mutex
 
-	features      map[string]string
+	Features map[string]string
 
-	startOfBench time.Time
-	endOfBench   time.Time
+	Start time.Time
+	End   time.Time
 
-	totalDuration atomic.Int64
-	transactionsCount int
+	TotalLatency      *atomic.Int64
+	TransactionsCount int
+	Name              string
+}
+
+func (r *Report) MarshalJSON() ([]byte, error) {
+	type Aux Report
+	return json.Marshal(struct {
+		Aux
+		TotalLatency int64
+	}{
+		Aux:          Aux(*r),
+		TotalLatency: r.TotalLatency.Load(),
+	})
 }
 
 func (r *Report) TPS() float64 {
-	return (float64(time.Duration(r.transactionsCount)) / float64(r.endOfBench.Sub(r.startOfBench))) * float64(time.Second)
+	return (float64(time.Duration(r.TransactionsCount)) / float64(r.End.Sub(r.Start))) * float64(time.Second)
 }
 
 func (r *Report) AverageDuration() time.Duration {
-	return time.Duration(r.totalDuration.Load()) * time.Millisecond / time.Duration(r.transactionsCount)
+	return time.Duration(r.TotalLatency.Load()) * time.Millisecond / time.Duration(r.TransactionsCount)
 }
 
 func (r *Report) registerTransactionLatency(latency time.Duration) {
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	r.transactionsCount++
-	r.totalDuration.Add(latency.Milliseconds())
+	r.TransactionsCount++
+	r.TotalLatency.Add(latency.Milliseconds())
+}
+
+func (r *Report) reset() {
+	r.TotalLatency = &atomic.Int64{}
+	r.TransactionsCount = 0
+	r.Start = time.Now()
 }
 
-func newReport(features map[string]string) *Report {
-	return &Report{
-		startOfBench: time.Now(),
-		features: features,
+func newReport(features map[string]string, name string) *Report {
+	ret := &Report{
+		Name: name,
+		Features:     features,
+		mu:           &sync.Mutex{},
 	}
+	ret.reset()
+	return ret
 }

test/performance/write_test.go

@@ -3,11 +3,14 @@
 package performance_test
 
 import (
+	"encoding/json"
 	"fmt"
+	"github.com/stretchr/testify/require"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/formancehq/go-libs/logging"
-	"github.com/stretchr/testify/require"
 )
 
 var scripts = map[string]func(int) (string, map[string]string){
@@ -54,14 +57,26 @@ send [USD/2 100] (
 
 func BenchmarkWrite(b *testing.B) {
 
-	// set default env factories if not defined (remote mode not used)
+	// Set default env factories if not defined (remote mode not used)
 	if len(envFactories) == 0 {
 		envFactories = map[string]EnvFactory{
 			"core":       NewCoreEnvFactory(pgServer),
 			"testserver": NewTestServerEnvFactory(pgServer),
 		}
 	}
 
-	err := New(b, envFactories, scripts).Run(logging.TestingContext())
-	require.NoError(b, err)
+	// Execute benchmarks
+	reports := New(b, envFactories, scripts).Run(logging.TestingContext())
+
+	// Write report
+	if reportFile != "" {
+		require.NoError(b, os.MkdirAll(filepath.Dir(reportFile), 0755))
+
+		f, err := os.Create(reportFile)
+		require.NoError(b, err)
+
+		enc := json.NewEncoder(f)
+		enc.SetIndent("", "  ")
+		require.NoError(b, enc.Encode(reports))
+	}
 }