Added benchmarking for rosetta server endpoints through check:perf co…

cmd/check_perf.go

@@ -0,0 +1,65 @@
+// Copyright 2022 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/coinbase/rosetta-cli/pkg/results"
+	t "github.com/coinbase/rosetta-cli/pkg/tester"
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	checkPerfCmd = &cobra.Command{
+		Use:   "check:perf",
+		Short: "Benchmark performance of time-critical endpoints of Asset Issuer's Rosetta Implementation",
+		Long: `This command can be used to benchmark the performance of time critical methods for a Rosetta server.
+This is useful for ensuring that there are no performance degradations in the rosetta-server.`,
+		RunE: runCheckPerfCmd,
+	}
+)
+
+func runCheckPerfCmd(_ *cobra.Command, _ []string) error {
+	ctx, cancel := context.WithCancel(Context)
+	defer cancel()
+	g, ctx := errgroup.WithContext(ctx)
+
+	TotalNumEndpoints := int64(Config.Perf.NumTimesToHitEndpoints) * (Config.Perf.EndBlock - Config.Perf.StartBlock)
+	perfRawStats := &results.CheckPerfRawStats{AccountBalanceEndpointTotalTime: -1, BlockEndpointTotalTime: -1}
+
+	fmt.Printf("Running Check:Perf for %s:%s for blocks %d-%d \n", Config.Network.Blockchain, Config.Network.Network, Config.Perf.StartBlock, Config.Perf.EndBlock)
+
+	fetcher, timer, elapsed := t.SetupBenchmarking(Config)
+	blockEndpointTimeConstraint := time.Duration(Config.Perf.BlockEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond
+	blockEndpointCtx, blockEndpointCancel := context.WithTimeout(ctx, blockEndpointTimeConstraint)
+	g.Go(func() error {
+		return t.BmarkBlock(blockEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats)
+	})
+	defer blockEndpointCancel()
+
+	fetcher, timer, elapsed = t.SetupBenchmarking(Config)
+	accountBalanceEndpointTimeConstraint := time.Duration(Config.Perf.AccountBalanceEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond
+	accountBalanceEndpointCtx, accountBalanceEndpointCancel := context.WithTimeout(ctx, accountBalanceEndpointTimeConstraint)
+	g.Go(func() error {
+		return t.BmarkAccountBalance(accountBalanceEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats)
+	})
+	defer accountBalanceEndpointCancel()
+
+	return results.ExitPerf(Config.Perf, g.Wait(), perfRawStats)
+}

cmd/root.go

@@ -240,6 +240,9 @@ default values.`,
 	// Utils
 	rootCmd.AddCommand(utilsAsserterConfigurationCmd)
 	rootCmd.AddCommand(utilsTrainZstdCmd)
+
+	// Benchmark commands
+	rootCmd.AddCommand(checkPerfCmd)
 }
 
 func initConfig() {

configuration/configuration.go

@@ -44,6 +44,19 @@ func DefaultDataConfiguration() *DataConfiguration {
 	}
 }
 
+// DefaultPerfConfiguration returns the default *CheckPerfConfiguration
+// for running `check:perf`.
+func DefaultPerfConfiguration() *CheckPerfConfiguration {
+	return &CheckPerfConfiguration{
+		StartBlock:                             10,
+		BlockEndpointTimeConstraintMs:          50000000,
+		AccountBalanceEndpointTimeConstraintMs: 50000000,
+		EndBlock:                               50,
+		NumTimesToHitEndpoints:                 1,
+		StatsOutputFile:                        "./check_perf_stats.json",
+	}
+}
+
 // DefaultConfiguration returns a *Configuration with the
 // EthereumNetwork, DefaultURL, DefaultTimeout,
 // DefaultConstructionConfiguration and DefaultDataConfiguration.
@@ -61,6 +74,24 @@ func DefaultConfiguration() *Configuration {
 	}
 }
 
+func populatePerfMissingFields(
+	perfConfig *CheckPerfConfiguration,
+) *CheckPerfConfiguration {
+	if perfConfig == nil {
+		return nil
+	}
+
+	if len(perfConfig.StatsOutputFile) == 0 {
+		perfConfig.StatsOutputFile = DefaultOutputFile
+	}
+
+	if perfConfig.NumTimesToHitEndpoints == 0 {
+		perfConfig.NumTimesToHitEndpoints = DefaultNumTimesToHitEndpoints
+	}
+
+	return perfConfig
+}
+
 func populateConstructionMissingFields(
 	constructionConfig *ConstructionConfiguration,
 ) *ConstructionConfiguration {
@@ -171,6 +202,7 @@ func populateMissingFields(config *Configuration) *Configuration {
 
 	config.Construction = populateConstructionMissingFields(config.Construction)
 	config.Data = populateDataMissingFields(config.Data)
+	config.Perf = populatePerfMissingFields(config.Perf)
 
 	return config
 }

configuration/types.go

@@ -61,6 +61,14 @@ const (
 	DefaultStatusPort                        = 9090
 	DefaultMaxReorgDepth                     = 100
 
+	// Check Perf Default Configs
+	DefaultStartBlock                             = 100
+	DefaultEndBlock                               = 10000
+	DefaultNumTimesToHitEndpoints                 = 50
+	DefaultOutputFile                             = "./check_perf_stats.json"
+	DefaultBlockEndpointTimeConstraintMs          = 5000
+	DefaultAccountBalanceEndpointTimeConstraintMs = 5000
+
 	// ETH Defaults
 	EthereumIDBlockchain = "Ethereum"
 	EthereumIDNetwork    = "Ropsten"
@@ -227,7 +235,7 @@ type DataConfiguration struct {
 	InactiveReconciliationConcurrency uint64 `json:"inactive_reconciliation_concurrency"`
 
 	// InactiveReconciliationFrequency is the number of blocks to wait between
-	// inactive reconiliations on each account.
+	// inactive reconciliations on each account.
 	InactiveReconciliationFrequency uint64 `json:"inactive_reconciliation_frequency"`
 
 	// LogBlocks is a boolean indicating whether to log processed blocks.
@@ -340,7 +348,7 @@ type DataConfiguration struct {
 }
 
 // Configuration contains all configuration settings for running
-// check:data or check:construction.
+// check:data, check:construction, or check:perf.
 type Configuration struct {
 	// Network is the *types.NetworkIdentifier where transactions should
 	// be constructed and where blocks should be synced to monitor
@@ -433,4 +441,29 @@ type Configuration struct {
 
 	Construction *ConstructionConfiguration `json:"construction"`
 	Data         *DataConfiguration         `json:"data"`
+	Perf         *CheckPerfConfiguration    `json:"perf"`
+}
+
+//********************//
+// Check Perf configs //
+//********************//
+type CheckPerfConfiguration struct {
+
+	// StartBlock is the starting block for running check:perf.
+	// If not provided, then this defaults to 0 (the genesis block)
+	StartBlock int64 `json:"start_block,omitempty"`
+
+	BlockEndpointTimeConstraintMs int64 `json:"block_endpoint_time_constraint_ms"`
+
+	AccountBalanceEndpointTimeConstraintMs int64 `json:"account_balance_endpoint_time_constraint_ms"`
+
+	// EndBlock is the ending block for running check:perf.
+	// Must be provided when running check:perf
+	EndBlock int64 `json:"end_block"`
+
+	// NumTimesToHitEndpoints is the number of times each rosetta-server endpoint will be benchmarked
+	NumTimesToHitEndpoints int `json:"num_times_to_hit_endpoints"`
+
+	// Location to output test results
+	StatsOutputFile string `json:"check_perf_output_dir"`
 }

examples/configuration/default.json

@@ -39,5 +39,6 @@
   "results_output_file": "",
   "pruning_disabled": false,
   "initial_balance_fetch_disabled": false
- }
+ },
+ "perf": null
 }

pkg/results/perf_results.go

@@ -0,0 +1,157 @@
+// Copyright 2022 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package results
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/coinbase/rosetta-cli/configuration"
+	"github.com/coinbase/rosetta-sdk-go/utils"
+	"github.com/olekukonko/tablewriter"
+)
+
+// Output writes *CheckPerfResults to the provided
+// path.
+func (c *CheckPerfStats) Output(path string) {
+	if len(path) > 0 {
+		writeErr := utils.SerializeAndWrite(path, c)
+		if writeErr != nil {
+			log.Printf("%s: unable to save results\n", writeErr.Error())
+		}
+	}
+}
+
+type CheckPerfRawStats struct {
+	BlockEndpointTotalTime          time.Duration
+	BlockEndpointNumErrors          int64
+	AccountBalanceEndpointTotalTime time.Duration
+	AccountBalanceNumErrors         int64
+}
+
+// CheckPerfStats contains interesting stats that
+// are counted while running the check:perf.
+type CheckPerfStats struct {
+	StartBlock                          int64 `json:"start_block"`
+	EndBlock                            int64 `json:"end_block"`
+	NumTimesHitEachEndpoint             int   `json:"num_times_hit_each_endpoint"`
+	AccountBalanceEndpointAverageTimeMs int64 `json:"account_balance_endpoint_average_time_ms"`
+	AccountBalanceEndpointTotalTimeMs   int64 `json:"account_balance_endpoint_total_time_ms"`
+	AccountBalanceEndpointNumErrors     int64 `json:"account_balance_endpoint_num_errors"`
+	BlockEndpointAverageTimeMs          int64 `json:"block_endpoint_average_time_ms"`
+	BlockEndpointTotalTimeMs            int64 `json:"block_endpoint_total_time_ms"`
+	BlockEndpointNumErrors              int64 `json:"block_endpoint_num_errors"`
+}
+
+// Print logs CheckPerfStats to the console.
+func (c *CheckPerfStats) Print() {
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetRowLine(true)
+	table.SetRowSeparator("-")
+	table.SetHeader([]string{"check:perf Stats", "Description", "Value"})
+	table.Append([]string{"Start Block", "The Starting Block", strconv.FormatInt(c.StartBlock, 10)})
+	table.Append([]string{"End Block", "The Ending Block", strconv.FormatInt(c.EndBlock, 10)})
+	table.Append([]string{"Num Times Each Endpoint", "Number of times that each endpoint was hit", strconv.FormatInt(int64(c.NumTimesHitEachEndpoint), 10)})
+	table.Append(
+		[]string{
+			"/Block Endpoint Total Time",
+			"Total elapsed time taken to fetch all blocks (ms)",
+			strconv.FormatInt(c.BlockEndpointTotalTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Block Endpoint Average Time",
+			"Average time taken to fetch each block (ms)",
+			strconv.FormatInt(c.BlockEndpointAverageTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Block Endpoint Num Errors",
+			"Total num errors occurred while fetching blocks",
+			strconv.FormatInt(c.BlockEndpointNumErrors, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Account/Balance Endpoint Average Time",
+			"Average time taken to fetch each account balance (ms)",
+			strconv.FormatInt(c.AccountBalanceEndpointAverageTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Account/Balance Endpoint Total Time",
+			"Total elapsed time taken to fetch all account balances (ms)",
+			strconv.FormatInt(c.AccountBalanceEndpointTotalTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Account/Balance Endpoint Num Errors",
+			"Total num errors occurred while fetching account balances",
+			strconv.FormatInt(c.AccountBalanceEndpointNumErrors, 10),
+		},
+	)
+
+	table.Render()
+}
+
+// ComputeCheckPerfStats returns a populated CheckPerfStats.
+func ComputeCheckPerfStats(
+	config *configuration.CheckPerfConfiguration,
+	rawStats *CheckPerfRawStats,
+) *CheckPerfStats {
+	totalNumEndpointsHit := (config.EndBlock - config.StartBlock) * int64(config.NumTimesToHitEndpoints)
+	stats := &CheckPerfStats{
+		BlockEndpointAverageTimeMs:          rawStats.BlockEndpointTotalTime.Milliseconds() / totalNumEndpointsHit,
+		BlockEndpointTotalTimeMs:            rawStats.BlockEndpointTotalTime.Milliseconds(),
+		BlockEndpointNumErrors:              rawStats.BlockEndpointNumErrors,
+		AccountBalanceEndpointAverageTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds() / totalNumEndpointsHit,
+		AccountBalanceEndpointTotalTimeMs:   rawStats.AccountBalanceEndpointTotalTime.Milliseconds(),
+		AccountBalanceEndpointNumErrors:     rawStats.AccountBalanceNumErrors,
+		StartBlock:                          config.StartBlock,
+		EndBlock:                            config.EndBlock,
+		NumTimesHitEachEndpoint:             config.NumTimesToHitEndpoints,
+	}
+
+	return stats
+}
+
+// ExitPerf exits check:perf, logs the test results to the console,
+// and to a provided output path.
+func ExitPerf(
+	config *configuration.CheckPerfConfiguration,
+	err error,
+	rawStats *CheckPerfRawStats,
+) error {
+	if err != nil {
+		log.Fatal(fmt.Errorf("Check:Perf Failed!: %w", err))
+	}
+
+	stats := ComputeCheckPerfStats(
+		config,
+		rawStats,
+	)
+
+	stats.Print()
+	stats.Output(config.StatsOutputFile)
+
+	return err
+}

pkg/tester/benchmark_utils.go

@@ -0,0 +1,24 @@
+// Copyright 2022 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tester
+
+import "time"
+
+func timerFactory() func() time.Duration {
+	start := time.Now()
+	return func() time.Duration {
+		return time.Since(start)
+	}
+}