diff --git a/go.mod b/go.mod index 124a74566..2be2c5e4b 100644 --- a/go.mod +++ b/go.mod @@ -23,12 +23,13 @@ require ( github.com/grafana/pyroscope-go/godeltaprof v0.1.8 github.com/invopop/jsonschema v0.12.0 github.com/jackc/pgx/v5 v5.7.1 + github.com/jamiealquiza/tachymeter v2.0.0+incompatible github.com/logrusorgru/aurora v2.0.3+incompatible github.com/nats-io/nats.go v1.37.0 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 + github.com/ory/dockertest/v3 v3.11.0 github.com/pborman/uuid v1.2.1 - github.com/pkg/errors v0.9.1 github.com/shomali11/xsql v0.0.0-20190608141458-bf76292144df github.com/spf13/cobra v1.8.1 github.com/spf13/pflag v1.0.5 @@ -36,17 +37,18 @@ require ( github.com/uptrace/bun v1.2.3 github.com/uptrace/bun/dialect/pgdialect v1.2.3 github.com/xeipuuv/gojsonschema v1.2.0 + github.com/xo/dburl v0.23.2 go.opentelemetry.io/otel v1.30.0 go.opentelemetry.io/otel/trace v1.30.0 go.uber.org/fx v1.22.2 go.uber.org/mock v0.4.0 golang.org/x/oauth2 v0.23.0 + golang.org/x/sync v0.8.0 ) require ( dario.cat/mergo v1.0.1 // indirect filippo.io/edwards25519 v1.1.0 // indirect - github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/IBM/sarama v1.43.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect @@ -87,7 +89,6 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-chi/chi v4.1.2+incompatible // indirect github.com/go-chi/render v1.0.3 // indirect - github.com/go-echarts/go-echarts/v2 v2.4.2 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -111,7 +112,6 @@ require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jamiealquiza/tachymeter v2.0.0+incompatible // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -136,14 +136,13 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.1.14 // indirect - github.com/ory/dockertest/v3 v3.11.0 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/riandyrn/otelchi v0.10.0 // indirect - github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481 // indirect github.com/rs/cors v1.11.1 // indirect github.com/shirou/gopsutil/v4 v4.24.8 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect @@ -164,7 +163,6 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect - github.com/xo/dburl v0.23.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zitadel/oidc/v2 v2.12.2 // indirect go.opentelemetry.io/contrib/instrumentation/host v0.55.0 // indirect @@ -188,7 +186,6 @@ require ( go.uber.org/zap v1.27.0 // indirect golang.org/x/crypto v0.27.0 // indirect golang.org/x/net v0.29.0 // indirect - golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.6.0 // indirect diff --git a/go.sum b/go.sum index 7a52cc1a9..fc652b0e4 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4 h1:tTl76UtV0KnHdcIxHAhq31NNO/B/r0+LnPsvUoIntk4= -github.com/Arafatk/glot v0.0.0-20230425001707-a00521b72ee4/go.mod h1:o0O8gFiTfVp4g5QcQJ1iMLw6ROiy9BITaiBbEiwz9h8= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/IBM/sarama v1.43.3 h1:Yj6L2IaNvb2mRBop39N7mmJAHBVY3dTPncr3qGVkxPA= @@ -95,12 +93,6 @@ github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/formancehq/go-libs v1.7.2-0.20240927143916-90a2f07df259 h1:UNR167W1JQgqqcHyvE6qRc0oYYN68s6zdee2bjlxfDA= -github.com/formancehq/go-libs v1.7.2-0.20240927143916-90a2f07df259/go.mod h1:oLMVltNsXmvqLA0Ox5NSwVSeH8gbnG2ZUo6Lchp23XU= -github.com/formancehq/go-libs v1.7.2-0.20240930151849-9e7ee54046e9 h1:l9ZGJmrgHPww1eyPhpo4jnv4Bjwz2o/Wvt0Fjh//gYg= -github.com/formancehq/go-libs v1.7.2-0.20240930151849-9e7ee54046e9/go.mod h1:oLMVltNsXmvqLA0Ox5NSwVSeH8gbnG2ZUo6Lchp23XU= -github.com/formancehq/go-libs v1.7.2-0.20241001142646-eb9c0c1cd58a h1:TkUCCiZjo7uQ1vY9E2RLaA2zxtEjnodSL9QILDyBoR0= -github.com/formancehq/go-libs v1.7.2-0.20241001142646-eb9c0c1cd58a/go.mod h1:oLMVltNsXmvqLA0Ox5NSwVSeH8gbnG2ZUo6Lchp23XU= github.com/formancehq/go-libs v1.7.2-0.20241001151743-2bd31525c7d1 h1:/Zo3e6njdSqIwi4nI038P2fTvzcfRQ6WaPmFzSI7s8o= github.com/formancehq/go-libs v1.7.2-0.20241001151743-2bd31525c7d1/go.mod h1:oLMVltNsXmvqLA0Ox5NSwVSeH8gbnG2ZUo6Lchp23XU= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -113,8 +105,6 @@ github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4= github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= -github.com/go-echarts/go-echarts/v2 v2.4.2 h1:1FC3tGzsLSgdeO4Ltc3OAtcIiRomfEKxKX9oocIL68g= -github.com/go-echarts/go-echarts/v2 v2.4.2/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -280,8 +270,6 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/riandyrn/otelchi v0.10.0 h1:QMbR/FMDWBOkej6dfyWteYefUKqIFxnyrpaoWRJ9RPQ= github.com/riandyrn/otelchi v0.10.0/go.mod h1:zBaX2FavWMlsvq4GqHit+QXxF1c5wIMZZFaYyW4+7FA= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481 h1:jMxcLa+VjJKhpCwbLUXAD15wJ+hhvXMLujCl3MkXpfM= -github.com/robertkrimen/godocdown v0.0.0-20130622164427-0bfa04905481/go.mod h1:C9WhFzY47SzYBIvzFqSvHIR6ROgDo4TtdTuRaOMjF/s= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= diff --git a/test/performance/benchmark_test.go b/test/performance/benchmark_test.go index e09597195..91aa8f074 100644 --- a/test/performance/benchmark_test.go +++ b/test/performance/benchmark_test.go @@ -16,30 +16,39 @@ import ( "github.com/stretchr/testify/require" ) +type TransactionProvider interface { + Get(iteration int) (string, map[string]string) +} +type TransactionProviderFn func(iteration int) (string, map[string]string) +func (fn TransactionProviderFn) Get(iteration int) (string, map[string]string) { + return fn(iteration) +} + type Benchmark struct { EnvFactory EnvFactory - Scripts map[string]func(int) (string, map[string]string) + Scenarios map[string]TransactionProvider - reports map[string]map[string]*Report + reports map[string]map[string]*report b *testing.B } -func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Report { - reports := make(map[string][]Report, 0) - scriptsKeys := Keys(benchmark.Scripts) - sort.Strings(scriptsKeys) +func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Result { + results := make(map[string][]Result, 0) + scenarios := Keys(benchmark.Scenarios) + sort.Strings(scenarios) - for _, scriptName := range scriptsKeys { + for _, scenario := range scenarios { for _, configuration := range buildAllPossibleConfigurations() { - testName := fmt.Sprintf("%s/%s", scriptName, configuration) + testName := fmt.Sprintf("%s/%s", scenario, configuration) ledgerConfiguration := ledger.Configuration{ Features: configuration.FeatureSet, Bucket: uuid.NewString()[:8], } ledgerConfiguration.SetDefaults() - report := newReport(configuration, scriptName) + report := newReport(configuration, scenario) + var result Result benchmark.b.Run(testName, func(b *testing.B) { report.reset() @@ -59,7 +68,7 @@ func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Report { for pb.Next() { iteration := int(cpt.Add(1)) - script, vars := benchmark.Scripts[scriptName](iteration) + script, vars := benchmark.Scenarios[scenario].Get(iteration) now := time.Now() _, err := env.Executor().ExecuteScript(ctx, script, vars) require.NoError(b, err) @@ -69,9 +78,10 @@ func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Report { }) b.StopTimer() report.End = time.Now() + result = report.GetResult() b.ReportMetric(report.TPS(), "t/s") - b.ReportMetric(float64(report.Tachymeter.Calc().Time.Avg.Milliseconds()), "ms/transaction") + b.ReportMetric(float64(result.Metrics.Time.Avg.Milliseconds()), "ms/transaction") stopContext, cancel := context.WithTimeout(ctx, 10*time.Second) b.Cleanup(cancel) @@ -80,19 +90,19 @@ func (benchmark *Benchmark) Run(ctx context.Context) map[string][]Report { }) if report.Tachymeter.Count > 0 { - reports[scriptName] = append(reports[scriptName], report) + results[scenario] = append(results[scenario], result) } } } - return reports + return results } -func New(b *testing.B, envFactory EnvFactory, scripts map[string]func(int) (string, map[string]string)) *Benchmark { +func New(b *testing.B, envFactory EnvFactory, scenarios map[string]TransactionProvider) *Benchmark { return &Benchmark{ b: b, EnvFactory: envFactory, - Scripts: scripts, - reports: make(map[string]map[string]*Report), + Scenarios: scenarios, + reports: make(map[string]map[string]*report), } } diff --git a/test/performance/charts/src/report.ts b/test/performance/charts/src/report.ts index def599bca..d66cfdfbf 100644 --- a/test/performance/charts/src/report.ts +++ b/test/performance/charts/src/report.ts @@ -37,7 +37,7 @@ interface Report { start: string, end: string, metrics: Metrics, - name: string, + scenario: string, configuration: Configuration, tps: number } diff --git a/test/performance/report_test.go b/test/performance/report_test.go index 511a99086..ff6431a81 100644 --- a/test/performance/report_test.go +++ b/test/performance/report_test.go @@ -3,13 +3,23 @@ package performance_test import ( - "encoding/json" "github.com/formancehq/go-libs/time" "github.com/jamiealquiza/tachymeter" "sync" ) -type Report struct { +type Result struct { + Start time.Time `json:"start"` + End time.Time `json:"end"` + + Metrics *tachymeter.Metrics `json:"metrics"` + + Name string `json:"name"` + Configuration configuration `json:"configuration"` + TPS float64 `json:"tps"` +} + +type report struct { mu *sync.Mutex Start time.Time @@ -17,50 +27,40 @@ type Report struct { Tachymeter *tachymeter.Tachymeter - Name string - Configuration configuration + Scenario string + Configuration configuration } -func (r Report) MarshalJSON() ([]byte, error) { - type view struct { - Start time.Time `json:"start"` - End time.Time `json:"end"` - - Metrics *tachymeter.Metrics `json:"metrics"` - - Name string `json:"name"` - Configuration configuration `json:"configuration"` - TPS float64 `json:"tps"` - } - return json.Marshal(view{ +func (r *report) GetResult() Result { + return Result{ Start: r.Start, End: r.End, Metrics: r.Tachymeter.Calc(), - Name: r.Name, + Name: r.Scenario, Configuration: r.Configuration, TPS: r.TPS(), - }) + } } -func (r *Report) TPS() float64 { +func (r *report) TPS() float64 { return (float64(time.Duration(r.Tachymeter.Count)) / float64(r.End.Sub(r.Start))) * float64(time.Second) } -func (r *Report) registerTransactionLatency(latency time.Duration) { +func (r *report) registerTransactionLatency(latency time.Duration) { r.mu.Lock() defer r.mu.Unlock() r.Tachymeter.AddTime(latency) } -func (r *Report) reset() { +func (r *report) reset() { r.Start = time.Now() r.Tachymeter.Reset() } -func newReport(configuration configuration, name string) Report { - ret := Report{ - Name: name, +func newReport(configuration configuration, scenario string) report { + ret := report{ + Scenario: scenario, Configuration: configuration, mu: &sync.Mutex{}, Tachymeter: tachymeter.New(&tachymeter.Config{ diff --git a/test/performance/write_test.go b/test/performance/write_test.go index 2e880dfb3..3abc91b5d 100644 --- a/test/performance/write_test.go +++ b/test/performance/write_test.go @@ -12,11 +12,11 @@ import ( "testing" ) -var scripts = map[string]func(int) (string, map[string]string){ - "world->bank": worldToBank, - "world->any": worldToAny, - "any(unbounded)->any": anyUnboundedToAny, - "any(bounded)->any": anyBoundedToAny, +var scripts = map[string]TransactionProvider{ + "world->bank": TransactionProviderFn(worldToBank), + "world->any": TransactionProviderFn(worldToAny), + "any(unbounded)->any": TransactionProviderFn(anyUnboundedToAny), + "any(bounded)->any": TransactionProviderFn(anyBoundedToAny), } func worldToBank(_ int) (string, map[string]string) {