diff --git a/test/performance/benchmark_test.go b/test/performance/benchmark_test.go index 4ecd61856..701560b57 100644 --- a/test/performance/benchmark_test.go +++ b/test/performance/benchmark_test.go @@ -24,7 +24,8 @@ type Benchmark struct { b *testing.B } -func (benchmark *Benchmark) Run(ctx context.Context) error { +func (benchmark *Benchmark) Run(ctx context.Context) []*Report { + reports := make([]*Report, 0) for envName, envFactory := range benchmark.EnvFactories { scriptsKeys := collectionutils.Keys(benchmark.Scripts) sort.Strings(scriptsKeys) @@ -34,19 +35,21 @@ func (benchmark *Benchmark) Run(ctx context.Context) error { testName := fmt.Sprintf("%s/%s/%s", envName, scriptName, features) + ledgerConfiguration := ledger.Configuration{ + Features: features, + Bucket: uuid.NewString()[:8], + } + ledgerConfiguration.SetDefaults() + report := newReport(ledgerConfiguration.Features, scriptName) + benchmark.b.Run(testName, func(b *testing.B) { - ledgerConfiguration := ledger.Configuration{ - Features: features, - Bucket: uuid.NewString()[:8], - } - ledgerConfiguration.SetDefaults() + report.reset() l := ledger.Ledger{ Configuration: ledgerConfiguration, Name: uuid.NewString()[:8], } cpt := atomic.Int64{} - report := newReport(ledgerConfiguration.Features) env := envFactory.Create(ctx, b, l) b.Logf("ledger: %s/%s", l.Bucket, l.Name) @@ -66,7 +69,7 @@ func (benchmark *Benchmark) Run(ctx context.Context) error { } }) b.StopTimer() - report.endOfBench = time.Now() + report.End = time.Now() b.ReportMetric(report.TPS(), "t/s") b.ReportMetric(float64(report.AverageDuration().Milliseconds()), "ms/transaction") @@ -76,11 +79,15 @@ func (benchmark *Benchmark) Run(ctx context.Context) error { require.NoError(benchmark.b, env.Stop(stopContext)) }) + + if report.TransactionsCount > 0 { + reports = append(reports, report) + } } } } - return nil + return reports } func New(b *testing.B, envFactories map[string]EnvFactory, scripts map[string]func(int) (string, map[string]string)) *Benchmark { diff --git a/test/performance/main_test.go b/test/performance/main_test.go index 3dc32dee1..8b5bccc9f 100644 --- a/test/performance/main_test.go +++ b/test/performance/main_test.go @@ -32,6 +32,7 @@ var ( ledgerURL string parallelism int64 + reportFile string envFactories = make(map[string]EnvFactory) ) @@ -42,6 +43,7 @@ func init() { flag.StringVar(&authClientSecret, "client.secret", "", "Client secret") flag.StringVar(&ledgerURL, "ledger.url", "", "Ledger url") flag.StringVar(&authIssuerURL, "auth.url", "", "Auth url (ignored if --stack.url is specified)") + flag.StringVar(&reportFile, "report.file", "", "Location to write report file") flag.Int64Var(¶llelism, "parallelism", 1, "Parallelism (default 1). Values is multiplied by GOMAXPROCS") } diff --git a/test/performance/report_test.go b/test/performance/report_test.go index 9a18f1f88..250437653 100644 --- a/test/performance/report_test.go +++ b/test/performance/report_test.go @@ -3,6 +3,7 @@ package performance_test import ( + "encoding/json" "sync" "sync/atomic" @@ -10,36 +11,57 @@ import ( ) type Report struct { - mu sync.Mutex + mu *sync.Mutex - features map[string]string + Features map[string]string - startOfBench time.Time - endOfBench time.Time + Start time.Time + End time.Time - totalDuration atomic.Int64 - transactionsCount int + TotalLatency *atomic.Int64 + TransactionsCount int + Name string +} + +func (r *Report) MarshalJSON() ([]byte, error) { + type Aux Report + return json.Marshal(struct { + Aux + TotalLatency int64 + }{ + Aux: Aux(*r), + TotalLatency: r.TotalLatency.Load(), + }) } func (r *Report) TPS() float64 { - return (float64(time.Duration(r.transactionsCount)) / float64(r.endOfBench.Sub(r.startOfBench))) * float64(time.Second) + return (float64(time.Duration(r.TransactionsCount)) / float64(r.End.Sub(r.Start))) * float64(time.Second) } func (r *Report) AverageDuration() time.Duration { - return time.Duration(r.totalDuration.Load()) * time.Millisecond / time.Duration(r.transactionsCount) + return time.Duration(r.TotalLatency.Load()) * time.Millisecond / time.Duration(r.TransactionsCount) } func (r *Report) registerTransactionLatency(latency time.Duration) { r.mu.Lock() defer r.mu.Unlock() - r.transactionsCount++ - r.totalDuration.Add(latency.Milliseconds()) + r.TransactionsCount++ + r.TotalLatency.Add(latency.Milliseconds()) +} + +func (r *Report) reset() { + r.TotalLatency = &atomic.Int64{} + r.TransactionsCount = 0 + r.Start = time.Now() } -func newReport(features map[string]string) *Report { - return &Report{ - startOfBench: time.Now(), - features: features, +func newReport(features map[string]string, name string) *Report { + ret := &Report{ + Name: name, + Features: features, + mu: &sync.Mutex{}, } + ret.reset() + return ret } diff --git a/test/performance/write_test.go b/test/performance/write_test.go index e144766d9..b3178c703 100644 --- a/test/performance/write_test.go +++ b/test/performance/write_test.go @@ -3,11 +3,14 @@ package performance_test import ( + "encoding/json" "fmt" + "github.com/stretchr/testify/require" + "os" + "path/filepath" "testing" "github.com/formancehq/go-libs/logging" - "github.com/stretchr/testify/require" ) var scripts = map[string]func(int) (string, map[string]string){ @@ -54,7 +57,7 @@ send [USD/2 100] ( func BenchmarkWrite(b *testing.B) { - // set default env factories if not defined (remote mode not used) + // Set default env factories if not defined (remote mode not used) if len(envFactories) == 0 { envFactories = map[string]EnvFactory{ "core": NewCoreEnvFactory(pgServer), @@ -62,6 +65,18 @@ func BenchmarkWrite(b *testing.B) { } } - err := New(b, envFactories, scripts).Run(logging.TestingContext()) - require.NoError(b, err) + // Execute benchmarks + reports := New(b, envFactories, scripts).Run(logging.TestingContext()) + + // Write report + if reportFile != "" { + require.NoError(b, os.MkdirAll(filepath.Dir(reportFile), 0755)) + + f, err := os.Create(reportFile) + require.NoError(b, err) + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + require.NoError(b, enc.Encode(reports)) + } }