From 6e5e6288d3c446c05618b6129e9bfe7a8b619159 Mon Sep 17 00:00:00 2001 From: raghavapamula Date: Tue, 10 May 2022 10:24:37 -0400 Subject: [PATCH] Added benchmarking for rosetta server endpoints through check:perf command Signed-off-by: raghavapamula --- cmd/check_perf.go | 65 ++++++++++++ cmd/root.go | 3 + configuration/configuration.go | 32 ++++++ configuration/types.go | 37 ++++++- examples/configuration/default.json | 3 +- pkg/results/perf_results.go | 157 ++++++++++++++++++++++++++++ pkg/tester/benchmark_utils.go | 24 +++++ pkg/tester/data_perf.go | 96 +++++++++++++++++ 8 files changed, 414 insertions(+), 3 deletions(-) create mode 100644 cmd/check_perf.go create mode 100644 pkg/results/perf_results.go create mode 100644 pkg/tester/benchmark_utils.go create mode 100644 pkg/tester/data_perf.go diff --git a/cmd/check_perf.go b/cmd/check_perf.go new file mode 100644 index 00000000..545d3b27 --- /dev/null +++ b/cmd/check_perf.go @@ -0,0 +1,65 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "time" + + "github.com/coinbase/rosetta-cli/pkg/results" + t "github.com/coinbase/rosetta-cli/pkg/tester" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +var ( + checkPerfCmd = &cobra.Command{ + Use: "check:perf", + Short: "Benchmark performance of time-critical endpoints of Asset Issuer's Rosetta Implementation", + Long: `This command can be used to benchmark the performance of time critical methods for a rosetta server. +This is useful for ensuring that there are no performance degradations in the rosetta-server.`, + RunE: runCheckPerfCmd, + } +) + +func runCheckPerfCmd(_ *cobra.Command, _ []string) error { + ctx, cancel := context.WithCancel(Context) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + + TotalNumEndpoints := int64(Config.Perf.NumTimesToHitEndpoints) * (Config.Perf.EndBlock - Config.Perf.StartBlock) + perfRawStats := &results.CheckPerfRawStats{AccountBalanceEndpointTotalTime: -1, BlockEndpointTotalTime: -1} + + fmt.Printf("Running Check:Perf for %s:%s for blocks %d-%d \n", Config.Network.Blockchain, Config.Network.Network, Config.Perf.StartBlock, Config.Perf.EndBlock) + + fetcher, timer, elapsed := t.Setup_Benchmarking(Config) + blockEndpointTimeConstraint := time.Duration(Config.Perf.BlockEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond + blockEndpointCtx, blockEndpointCancel := context.WithTimeout(ctx, blockEndpointTimeConstraint) + g.Go(func() error { + return t.Bmark_Block(blockEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats) + }) + defer blockEndpointCancel() + + fetcher, timer, elapsed = t.Setup_Benchmarking(Config) + accountBalanceEndpointTimeConstraint := time.Duration(Config.Perf.AccountBalanceEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond + accountBalanceEndpointCtx, accountBalanceEndpointCancel := context.WithTimeout(ctx, accountBalanceEndpointTimeConstraint) + g.Go(func() error { + return t.Bmark_AccountBalance(accountBalanceEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats) + }) + defer accountBalanceEndpointCancel() + + return results.ExitPerf(Config.Perf, g.Wait(), perfRawStats) +} diff --git a/cmd/root.go b/cmd/root.go index 649bd2f6..1c41844d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -240,6 +240,9 @@ default values.`, // Utils rootCmd.AddCommand(utilsAsserterConfigurationCmd) rootCmd.AddCommand(utilsTrainZstdCmd) + + // Benchmark commands + rootCmd.AddCommand(checkPerfCmd) } func initConfig() { diff --git a/configuration/configuration.go b/configuration/configuration.go index c92c2cd2..3b685a4e 100644 --- a/configuration/configuration.go +++ b/configuration/configuration.go @@ -44,6 +44,19 @@ func DefaultDataConfiguration() *DataConfiguration { } } +// DefaultPerfConfiguration returns the default *CheckPerfConfiguration +// for running `check:perf`. +func DefaultPerfConfiguration() *CheckPerfConfiguration { + return &CheckPerfConfiguration{ + StartBlock: 10, + BlockEndpointTimeConstraintMs: 50000000, + AccountBalanceEndpointTimeConstraintMs: 50000000, + EndBlock: 50, + NumTimesToHitEndpoints: 1, + StatsOutputFile: "./check_perf_stats.json", + } +} + // DefaultConfiguration returns a *Configuration with the // EthereumNetwork, DefaultURL, DefaultTimeout, // DefaultConstructionConfiguration and DefaultDataConfiguration. @@ -61,6 +74,24 @@ func DefaultConfiguration() *Configuration { } } +func populatePerfMissingFields( + perfConfig *CheckPerfConfiguration, +) *CheckPerfConfiguration { + if perfConfig == nil { + return nil + } + + if len(perfConfig.StatsOutputFile) == 0 { + perfConfig.StatsOutputFile = DefaultOutputFile + } + + if perfConfig.NumTimesToHitEndpoints == 0 { + perfConfig.NumTimesToHitEndpoints = DefaultNumTimesToHitEndpoints + } + + return perfConfig +} + func populateConstructionMissingFields( constructionConfig *ConstructionConfiguration, ) *ConstructionConfiguration { @@ -171,6 +202,7 @@ func populateMissingFields(config *Configuration) *Configuration { config.Construction = populateConstructionMissingFields(config.Construction) config.Data = populateDataMissingFields(config.Data) + config.Perf = populatePerfMissingFields(config.Perf) return config } diff --git a/configuration/types.go b/configuration/types.go index 1044143b..603afbef 100644 --- a/configuration/types.go +++ b/configuration/types.go @@ -61,6 +61,14 @@ const ( DefaultStatusPort = 9090 DefaultMaxReorgDepth = 100 + // Check Perf Default Configs + DefaultStartBlock = 100 + DefaultEndBlock = 10000 + DefaultNumTimesToHitEndpoints = 50 + DefaultOutputFile = "./check_perf_stats.json" + DefaultBlockEndpointTimeConstraintMs = 5000 + DefaultAccountBalanceEndpointTimeConstraintMs = 5000 + // ETH Defaults EthereumIDBlockchain = "Ethereum" EthereumIDNetwork = "Ropsten" @@ -227,7 +235,7 @@ type DataConfiguration struct { InactiveReconciliationConcurrency uint64 `json:"inactive_reconciliation_concurrency"` // InactiveReconciliationFrequency is the number of blocks to wait between - // inactive reconiliations on each account. + // inactive reconciliations on each account. InactiveReconciliationFrequency uint64 `json:"inactive_reconciliation_frequency"` // LogBlocks is a boolean indicating whether to log processed blocks. @@ -340,7 +348,7 @@ type DataConfiguration struct { } // Configuration contains all configuration settings for running -// check:data or check:construction. +// check:data, check:construction, or check:perf. type Configuration struct { // Network is the *types.NetworkIdentifier where transactions should // be constructed and where blocks should be synced to monitor @@ -433,4 +441,29 @@ type Configuration struct { Construction *ConstructionConfiguration `json:"construction"` Data *DataConfiguration `json:"data"` + Perf *CheckPerfConfiguration `json:"perf"` +} + +//********************// +// Check Perf configs // +//********************// +type CheckPerfConfiguration struct { + + // StartBlock is the starting block for running check:perf. + // If not provided, then this defaults to 0 (the genesis block) + StartBlock int64 `json:"start_block,omitempty"` + + BlockEndpointTimeConstraintMs int64 `json:"block_endpoint_time_constraint_ms"` + + AccountBalanceEndpointTimeConstraintMs int64 `json:"account_balance_endpoint_time_constraint_ms"` + + // EndBlock is the ending block for running check:perf. + // Must be provided when running check:perf + EndBlock int64 `json:"end_block"` + + // NumTimesToHitEndpoints is the number of times each rosetta-server endpoint will be benchmarked + NumTimesToHitEndpoints int `json:"num_times_to_hit_endpoints"` + + // Location to output test results + StatsOutputFile string `json:"check_perf_output_dir"` } diff --git a/examples/configuration/default.json b/examples/configuration/default.json index 11ee99f3..df5c68ab 100644 --- a/examples/configuration/default.json +++ b/examples/configuration/default.json @@ -39,5 +39,6 @@ "results_output_file": "", "pruning_disabled": false, "initial_balance_fetch_disabled": false - } + }, + "perf": null } \ No newline at end of file diff --git a/pkg/results/perf_results.go b/pkg/results/perf_results.go new file mode 100644 index 00000000..30732994 --- /dev/null +++ b/pkg/results/perf_results.go @@ -0,0 +1,157 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package results + +import ( + "fmt" + "log" + "os" + "strconv" + "time" + + "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-sdk-go/utils" + "github.com/olekukonko/tablewriter" +) + +// Output writes *CheckPerfResults to the provided +// path. +func (c *CheckPerfStats) Output(path string) { + if len(path) > 0 { + writeErr := utils.SerializeAndWrite(path, c) + if writeErr != nil { + log.Printf("%s: unable to save results\n", writeErr.Error()) + } + } +} + +type CheckPerfRawStats struct { + BlockEndpointTotalTime time.Duration + BlockEndpointNumErrors int64 + AccountBalanceEndpointTotalTime time.Duration + AccountBalanceNumErrors int64 +} + +// CheckPerfStats contains interesting stats that +// are counted while running the check:perf. +type CheckPerfStats struct { + StartBlock int64 `json:"start_block"` + EndBlock int64 `json:"end_block"` + NumTimesHitEachEndpoint int `json:"num_times_hit_each_endpoint"` + AccountBalanceEndpointAverageTimeMs int64 `json:"account_balance_endpoint_average_time_ms"` + AccountBalanceEndpointTotalTimeMs int64 `json:"account_balance_endpoint_total_time_ms"` + AccountBalanceEndpointNumErrors int64 `json:"account_balance_endpoint_num_errors"` + BlockEndpointAverageTimeMs int64 `json:"block_endpoint_average_time_ms"` + BlockEndpointTotalTimeMs int64 `json:"block_endpoint_total_time_ms"` + BlockEndpointNumErrors int64 `json:"block_endpoint_num_errors"` +} + +// Print logs CheckPerfStats to the console. +func (c *CheckPerfStats) Print() { + table := tablewriter.NewWriter(os.Stdout) + table.SetRowLine(true) + table.SetRowSeparator("-") + table.SetHeader([]string{"check:perf Stats", "Description", "Value"}) + table.Append([]string{"Start Block", "Start Block", strconv.FormatInt(c.StartBlock, 10)}) + table.Append([]string{"End Block", "EndBlock", strconv.FormatInt(c.EndBlock, 10)}) + table.Append([]string{"Num Times Each Endpoint", "Number of times that each endpoint was hit", strconv.FormatInt(int64(c.NumTimesHitEachEndpoint), 10)}) + table.Append( + []string{ + "/Block Endpoint Total Time", + "Total elapsed time taken to fetch all blocks (ms)", + strconv.FormatInt(c.BlockEndpointTotalTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Block Endpoint Average Time", + "Average time taken to fetch each block (ms)", + strconv.FormatInt(c.BlockEndpointAverageTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Block Endpoint Num Errors", + "Total num errors occurred while fetching blocks", + strconv.FormatInt(c.BlockEndpointNumErrors, 10), + }, + ) + table.Append( + []string{ + "/Account/Balance Endpoint Average Time", + "Average time taken to fetch each account balance (ms)", + strconv.FormatInt(c.AccountBalanceEndpointAverageTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Account/Balance Endpoint Total Time", + "Total elapsed time taken to fetch all account balances (ms)", + strconv.FormatInt(c.AccountBalanceEndpointTotalTimeMs, 10), + }, + ) + table.Append( + []string{ + "/Account/Balance Endpoint Num Errors", + "Total num errors occurred while fetching account balances", + strconv.FormatInt(c.AccountBalanceEndpointNumErrors, 10), + }, + ) + + table.Render() +} + +// ComputeCheckPerfStats returns a populated CheckPerfStats. +func ComputeCheckPerfStats( + config *configuration.CheckPerfConfiguration, + rawStats *CheckPerfRawStats, +) *CheckPerfStats { + totalNumEndpointsHit := (config.EndBlock - config.StartBlock) * int64(config.NumTimesToHitEndpoints) + stats := &CheckPerfStats{ + BlockEndpointAverageTimeMs: rawStats.BlockEndpointTotalTime.Milliseconds() / totalNumEndpointsHit, + BlockEndpointTotalTimeMs: rawStats.BlockEndpointTotalTime.Milliseconds(), + BlockEndpointNumErrors: rawStats.BlockEndpointNumErrors, + AccountBalanceEndpointAverageTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds() / totalNumEndpointsHit, + AccountBalanceEndpointTotalTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds(), + AccountBalanceEndpointNumErrors: rawStats.AccountBalanceNumErrors, + StartBlock: config.StartBlock, + EndBlock: config.EndBlock, + NumTimesHitEachEndpoint: config.NumTimesToHitEndpoints, + } + + return stats +} + +// ExitPerf exits check:perf, logs the test results to the console, +// and to a provided output path. +func ExitPerf( + config *configuration.CheckPerfConfiguration, + err error, + rawStats *CheckPerfRawStats, +) error { + if err != nil { + log.Fatal(fmt.Errorf("Check:Perf Failed!: %w", err)) + } + + stats := ComputeCheckPerfStats( + config, + rawStats, + ) + + stats.Print() + stats.Output(config.StatsOutputFile) + + return err +} diff --git a/pkg/tester/benchmark_utils.go b/pkg/tester/benchmark_utils.go new file mode 100644 index 00000000..c7962d09 --- /dev/null +++ b/pkg/tester/benchmark_utils.go @@ -0,0 +1,24 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import "time" + +func timerFactory() func() time.Duration { + start := time.Now() + return func() time.Duration { + return time.Since(start) + } +} diff --git a/pkg/tester/data_perf.go b/pkg/tester/data_perf.go new file mode 100644 index 00000000..5a789aca --- /dev/null +++ b/pkg/tester/data_perf.go @@ -0,0 +1,96 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tester + +import ( + "context" + "errors" + "time" + + "github.com/coinbase/rosetta-cli/configuration" + "github.com/coinbase/rosetta-cli/pkg/results" + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" +) + +func Setup_Benchmarking(config *configuration.Configuration) (*fetcher.Fetcher, func() time.Duration, chan time.Duration) { + // Create a new fetcher + fetcher := fetcher.New( + config.OnlineURL, + fetcher.WithMaxRetries(0), + ) + timer := timerFactory() + elapsed := make(chan time.Duration, 1) + return fetcher, timer, elapsed +} + +// Benchmark the asset issuer's /block endpoint +func Bmark_Block(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error { + total_errors := 0 + go func() { + for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ { + for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ { + partialBlockId := &types.PartialBlockIdentifier{ + Hash: nil, + Index: &m, + } + _, err := fetcher.Block(ctx, config.Network, partialBlockId) + if err != nil { + total_errors++ + } + } + } + elapsed <- timer() + }() + select { + case <-ctx.Done(): + return errors.New("/block endpoint benchmarking timed out") + case timeTaken := <-elapsed: + rawStats.BlockEndpointTotalTime = timeTaken + rawStats.BlockEndpointNumErrors = int64(total_errors) + return nil + } +} + +// Benchmark the asset issuers /account/balance endpoint +func Bmark_AccountBalance(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error { + total_errors := 0 + go func() { + for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ { + for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ { + account := &types.AccountIdentifier{ + Address: "address", + } + partialBlockId := &types.PartialBlockIdentifier{ + Hash: nil, + Index: &m, + } + _, _, _, err := fetcher.AccountBalance(ctx, config.Network, account, partialBlockId, nil) + if err != nil { + total_errors++ + } + } + } + elapsed <- timer() + }() + select { + case <-ctx.Done(): + return errors.New("/account/balance endpoint benchmarking timed out") + case timeTaken := <-elapsed: + rawStats.AccountBalanceEndpointTotalTime = timeTaken + rawStats.AccountBalanceNumErrors = int64(total_errors) + return nil + } +}