Merge pull request #201 from sujitdmello/master

Added a skipFirst option to allow the first X requests to be discounted from the result calculations

.github/workflows/build.yaml

@@ -31,7 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       GO111MODULE: on
-      GOLANGCI_LINT_VERSION: v1.22.0
+      GOLANGCI_LINT_VERSION: v1.27.0
 
     steps:
     - name: Set up Go

README.md

@@ -70,6 +70,7 @@ Flags:
       --key=                   File containing client private key, to present to the server. Must also provide -cert option.
       --cname=                 Server name override when validating TLS certificate - useful for self signed certs.
       --skipTLS                Skip TLS client verification of the server's certificate chain and host name.
+      --skipFirst=0            Skip the first X requests from the timing calculations (useful for initial warmup)
       --insecure               Use plaintext and insecure connection.
       --authority=             Value to be used as the :authority pseudo-header. Only works if -insecure is used.
   -c, --concurrency=50         Number of requests to run concurrently. Total number of requests cannot be smaller than the concurrency level. Default is 50.

cmd/ghz/main.go

@@ -65,6 +65,10 @@ var (
 	skipVerify = kingpin.Flag("skipTLS", "Skip TLS client verification of the server's certificate chain and host name.").
 			Default("false").IsSetByUser(&isSkipSet).Bool()
 
+	isSkipFirstSet = false
+	skipFirst      = kingpin.Flag("skipFirst", "Skip the first X requests when doing the results tally.").
+			Default("0").IsSetByUser(&isSkipFirstSet).Uint()
+
 	isInsecSet = false
 	insecure   = kingpin.Flag("insecure", "Use plaintext and insecure connection.").
 			Default("false").IsSetByUser(&isInsecSet).Bool()
@@ -357,6 +361,7 @@ func createConfigFromArgs(cfg *runner.Config) error {
 	cfg.Cert = *cert
 	cfg.Key = *key
 	cfg.SkipTLSVerify = *skipVerify
+	cfg.SkipFirst = *skipFirst
 	cfg.Insecure = *insecure
 	cfg.Authority = *authority
 	cfg.CName = *cname
@@ -423,6 +428,10 @@ func mergeConfig(dest *runner.Config, src *runner.Config) error {
 		dest.SkipTLSVerify = src.SkipTLSVerify
 	}
 
+	if isSkipFirstSet {
+		dest.SkipFirst = src.SkipFirst
+	}
+
 	if isInsecSet {
 		dest.Insecure = src.Insecure
 	}

go.mod

@@ -17,18 +17,24 @@ require (
 	github.com/jhump/protoreflect v1.5.0
 	github.com/jinzhu/configor v1.1.1
 	github.com/jinzhu/gorm v1.9.11
+	github.com/kr/text v0.2.0 // indirect
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/gommon v0.3.0
 	github.com/leodido/go-urn v1.2.0 // indirect
+	github.com/mattn/go-colorable v0.1.4 // indirect
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/pkg/errors v0.8.1
 	github.com/rakyll/statik v0.1.6
-	github.com/stretchr/testify v1.4.0
+	github.com/stretchr/testify v1.5.1
 	go.uber.org/multierr v1.3.0
 	go.uber.org/zap v1.13.0
-	golang.org/x/net v0.0.0-20191021144547-ec77196f6094
-	golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f // indirect
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b
+	golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e // indirect
+	golang.org/x/tools v0.0.0-20200502202811-ed308ab3e770 // indirect
 	google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a // indirect
 	google.golang.org/grpc v1.24.0
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
-	gopkg.in/yaml.v2 v2.2.4 // indirect
+	gopkg.in/yaml.v2 v2.2.8 // indirect
+	honnef.co/go/tools v0.0.1-2020.1.3 // indirect
 )

runner/config.go

@@ -63,6 +63,7 @@ type Config struct {
 	Cert              string            `json:"cert" toml:"cert" yaml:"cert"`
 	Key               string            `json:"key" toml:"key" yaml:"key"`
 	SkipTLSVerify     bool              `json:"skipTLS" toml:"skipTLS" yaml:"skipTLS"`
+	SkipFirst         uint              `json:"skipFirst" toml:"skipFirst" yaml:"skipFirst"`
 	CName             string            `json:"cname" toml:"cname" yaml:"cname"`
 	Authority         string            `json:"authority" toml:"authority" yaml:"authority"`
 	Insecure          bool              `json:"insecure,omitempty" toml:"insecure,omitempty" yaml:"insecure,omitempty"`

runner/options.go

@@ -66,9 +66,10 @@ type RunConfig struct {
 	log    Logger
 
 	// misc
-	name string
-	cpus int
-	tags []byte
+	name      string
+	cpus      int
+	tags      []byte
+	skipFirst int
 }
 
 // Option controls some aspect of run
@@ -450,6 +451,17 @@ func WithCPUs(c uint) Option {
 	}
 }
 
+// WithSkipFirst is the skipFirst option
+func WithSkipFirst(c uint) Option {
+	return func(o *RunConfig) error {
+		if c > 0 {
+			o.skipFirst = int(c)
+		}
+
+		return nil
+	}
+}
+
 // WithProtoFile specified proto file path and optionally import paths
 // We will automatically add the proto file path's directory and the current directory
 //	WithProtoFile("greeter.proto", []string{"/home/protos"})
@@ -458,7 +470,7 @@ func WithProtoFile(proto string, importPaths []string) Option {
 		proto = strings.TrimSpace(proto)
 		if proto != "" {
 			if filepath.Ext(proto) != ".proto" {
-				return errors.Errorf(fmt.Sprintf("proto: must have .proto extension"))
+				return errors.New("proto: must have .proto extension")
 			}
 
 			o.proto = proto
@@ -573,6 +585,9 @@ func newConfig(call, host string, options ...Option) (*RunConfig, error) {
 		return nil, errors.New("Host required")
 	}
 
+	if c.skipFirst > 0 && int(c.skipFirst) > c.n {
+		return nil, errors.New("You cannot skip more requests than those run")
+	}
 	creds, err := createClientTransportCredentials(
 		c.skipVerify,
 		c.cacert,
@@ -648,6 +663,7 @@ func fromConfig(cfg *Config) []Option {
 		WithCertificate(cfg.Cert, cfg.Key),
 		WithServerNameOverride(cfg.CName),
 		WithSkipTLSVerify(cfg.SkipTLSVerify),
+		WithSkipFirst(cfg.SkipFirst),
 		WithInsecure(cfg.Insecure),
 		WithAuthority(cfg.Authority),
 		WithConcurrency(cfg.C),

runner/options_test.go

@@ -48,6 +48,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 50, c.c)
 		assert.Equal(t, 0, c.qps)
 		assert.Equal(t, false, c.binary)
+		assert.Equal(t, 0, c.skipFirst)
 		assert.Equal(t, time.Duration(0), c.z)
 		assert.Equal(t, time.Duration(0), c.keepaliveTime)
 		assert.Equal(t, time.Duration(20*time.Second), c.timeout)
@@ -70,6 +71,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 			WithTotalRequests(100),
 			WithConcurrency(20),
 			WithQPS(5),
+			WithSkipFirst(5),
 			WithRunDuration(time.Duration(5*time.Minute)),
 			WithKeepalive(time.Duration(60*time.Second)),
 			WithTimeout(time.Duration(10*time.Second)),
@@ -89,6 +91,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 100, c.n)
 		assert.Equal(t, 20, c.c)
 		assert.Equal(t, 5, c.qps)
+		assert.Equal(t, 5, c.skipFirst)
 		assert.Equal(t, false, c.binary)
 		assert.Equal(t, time.Duration(5*time.Minute), c.z)
 		assert.Equal(t, time.Duration(60*time.Second), c.keepaliveTime)
@@ -114,6 +117,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 			WithTotalRequests(100),
 			WithConcurrency(20),
 			WithQPS(5),
+			WithSkipFirst(5),
 			WithRunDuration(time.Duration(5*time.Minute)),
 			WithKeepalive(time.Duration(60*time.Second)),
 			WithTimeout(time.Duration(10*time.Second)),
@@ -137,6 +141,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 100, c.n)
 		assert.Equal(t, 20, c.c)
 		assert.Equal(t, 5, c.qps)
+		assert.Equal(t, 5, c.skipFirst)
 		assert.Equal(t, true, c.binary)
 		assert.Equal(t, time.Duration(5*time.Minute), c.z)
 		assert.Equal(t, time.Duration(60*time.Second), c.keepaliveTime)
@@ -183,6 +188,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 			WithTotalRequests(100),
 			WithConcurrency(20),
 			WithQPS(5),
+			WithSkipFirst(5),
 			WithRunDuration(time.Duration(5*time.Minute)),
 			WithKeepalive(time.Duration(60*time.Second)),
 			WithTimeout(time.Duration(10*time.Second)),
@@ -205,6 +211,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 100, c.n)
 		assert.Equal(t, 20, c.c)
 		assert.Equal(t, 5, c.qps)
+		assert.Equal(t, 5, c.skipFirst)
 		assert.Equal(t, false, c.binary)
 		assert.Equal(t, time.Duration(5*time.Minute), c.z)
 		assert.Equal(t, time.Duration(60*time.Second), c.keepaliveTime)
@@ -237,6 +244,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 200, c.n)
 		assert.Equal(t, 50, c.c)
 		assert.Equal(t, 0, c.qps)
+		assert.Equal(t, 0, c.skipFirst)
 		assert.Equal(t, time.Duration(0), c.z)
 		assert.Equal(t, time.Duration(0), c.keepaliveTime)
 		assert.Equal(t, time.Duration(20*time.Second), c.timeout)
@@ -266,6 +274,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 200, c.n)
 		assert.Equal(t, 50, c.c)
 		assert.Equal(t, 0, c.qps)
+		assert.Equal(t, 0, c.skipFirst)
 		assert.Equal(t, false, c.binary)
 		assert.Equal(t, time.Duration(0), c.z)
 		assert.Equal(t, time.Duration(0), c.keepaliveTime)
@@ -300,6 +309,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 200, c.n)
 		assert.Equal(t, 50, c.c)
 		assert.Equal(t, 0, c.qps)
+		assert.Equal(t, 0, c.skipFirst)
 		assert.Equal(t, 1, c.nConns)
 		assert.Equal(t, false, c.binary)
 		assert.Equal(t, time.Duration(0), c.z)
@@ -336,6 +346,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, 200, c.n)
 		assert.Equal(t, 50, c.c)
 		assert.Equal(t, 0, c.qps)
+		assert.Equal(t, 0, c.skipFirst)
 		assert.Equal(t, 5, c.nConns)
 		assert.Equal(t, false, c.binary)
 		assert.Equal(t, time.Duration(0), c.z)
@@ -381,6 +392,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 			assert.Equal(t, []string{"../../testdata", "."}, c.importPaths)
 			assert.Equal(t, 5000, c.n)
 			assert.Equal(t, 50, c.c)
+			assert.Equal(t, 5, c.skipFirst)
 			assert.Equal(t, 12*time.Second, c.z)
 			assert.Equal(t, 500*time.Millisecond, c.streamInterval)
 			assert.Equal(t, []byte(`{"name":"Bob {{.TimestampUnix}}"}`), c.data)
@@ -399,6 +411,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 			assert.Equal(t, []string{"../../testdata", "."}, c.importPaths)
 			assert.Equal(t, 5000, c.n)
 			assert.Equal(t, 50, c.c)
+			assert.Equal(t, 5, c.skipFirst)
 			assert.Equal(t, 12*time.Second, c.z)
 			assert.Equal(t, 500*time.Millisecond, c.streamInterval)
 			assert.Equal(t, []byte(`{"name":"Bob {{.TimestampUnix}}"}`), c.data)
@@ -417,6 +430,7 @@ func TestRunConfig_newRunConfig(t *testing.T) {
 		assert.Equal(t, []string{"../../testdata", "."}, c.importPaths)
 		assert.Equal(t, 5000, c.n)
 		assert.Equal(t, 50, c.c)
+		assert.Equal(t, 5, c.skipFirst)
 		assert.Equal(t, 12*time.Second, c.z)
 		assert.Equal(t, 500*time.Millisecond, c.streamInterval)
 		assert.Equal(t, []byte(`{"name":"Bob {{.TimestampUnix}}"}`), c.data)

runner/reporter.go

@@ -33,6 +33,7 @@ type Options struct {
 	Cert          string        `json:"cert,omitempty"`
 	Key           string        `json:"key,omitempty"`
 	SkipTLS       bool          `json:"skipTLS,omitempty"`
+	SkipFirst     uint          `json:"skipFirst,omitempty"`
 	CName         string        `json:"cname,omitempty"`
 	Authority     string        `json:"authority,omitempty"`
 	Insecure      bool          `json:"insecure"`
@@ -133,9 +134,14 @@ func newReporter(results chan *callResult, c *RunConfig) *Reporter {
 
 // Run runs the reporter
 func (r *Reporter) Run() {
+	var skipCount int
+
 	for res := range r.results {
+		if skipCount < r.config.skipFirst {
+			skipCount++
+			continue
+		}
 		errStr := ""
-
 		r.totalCount++
 		r.totalLatenciesSec += res.duration.Seconds()
 		r.statusCodeDist[res.status]++
@@ -179,6 +185,7 @@ func (r *Reporter) Finalize(stopReason StopReason, total time.Duration) *Report
 		Key:           r.config.key,
 		CName:         r.config.cname,
 		SkipTLS:       r.config.skipVerify,
+		SkipFirst:     uint(r.config.skipFirst),
 		Insecure:      r.config.insecure,
 		Authority:     r.config.authority,
 		Total:         uint(r.config.n),

runner/run_test.go

@@ -66,6 +66,51 @@ func TestRunUnary(t *testing.T) {
 		assert.Equal(t, 1, count)
 	})
 
+	t.Run("test skip first N", func(t *testing.T) {
+		gs.ResetCounters()
+
+		data := make(map[string]interface{})
+		data["name"] = "bob"
+
+		report, err := Run(
+			"helloworld.Greeter.SayHello",
+			internal.TestLocalhost,
+			WithProtoFile("../testdata/greeter.proto", []string{}),
+			WithTotalRequests(10),
+			WithConcurrency(1),
+			WithTimeout(time.Duration(20*time.Second)),
+			WithDialTimeout(time.Duration(20*time.Second)),
+			WithData(data),
+			WithSkipFirst(5),
+			WithInsecure(true),
+		)
+
+		assert.NoError(t, err)
+
+		assert.NotNil(t, report)
+
+		assert.Equal(t, 5, int(report.Count))
+		assert.NotZero(t, report.Average)
+		assert.NotZero(t, report.Fastest)
+		assert.NotZero(t, report.Slowest)
+		assert.NotZero(t, report.Rps)
+		assert.Empty(t, report.Name)
+		assert.NotEmpty(t, report.Date)
+		assert.NotEmpty(t, report.Options)
+		assert.NotEmpty(t, report.Details)
+		assert.Equal(t, true, report.Options.Insecure)
+		assert.NotEmpty(t, report.LatencyDistribution)
+		assert.Equal(t, ReasonNormalEnd, report.EndReason)
+		assert.Empty(t, report.ErrorDist)
+
+		assert.NotEqual(t, report.Average, report.Slowest)
+		assert.NotEqual(t, report.Average, report.Fastest)
+		assert.NotEqual(t, report.Slowest, report.Fastest)
+
+		count := gs.GetCount(callType)
+		assert.Equal(t, 10, count)
+	})
+
 	t.Run("test N and Name", func(t *testing.T) {
 		gs.ResetCounters()
 

testdata/config.json

@@ -1,6 +1,7 @@
 {
   "total": 5000,
   "concurrency": 50,
+  "skipFirst": 5,
   "max-duration": "7s",
   "duration": "12s",
   "stream-interval": "500ms",

testdata/config.toml

@@ -1,6 +1,7 @@
 "max-duration" = "7s"
 total = 5000
 concurrency = 50
+skipFirst = 5
 proto = "../../testdata/greeter.proto"
 call = "helloworld.Greeter.SayHello"
 host = "0.0.0.0:50051"

www/docs/options.md

@@ -216,4 +216,8 @@ Show context-sensitive help (also try --help-long and --help-man).
 
 ### `-e`, `--enable-compression`               
 
-Enable gzip compression on requests.
+Enable gzip compression on requests.
+
+### '--skipFirst'
+
+Skip the first n responses from the report. Helps remove initial warm-up requests from skewing the results.

www/docs/usage.md

@@ -43,6 +43,7 @@ Flags:
       --name=                  User specified name for the test.
       --tags=                  JSON representation of user-defined string tags.
       --cpus=12                Number of cpu cores to use.
+      --skipFirst=0            Skip the first n responses from the report. Helps remove initial warm-up requests from skewing the results.
       --debug=                 The path to debug log file.
   -e, --enable-compression     Enable Gzip compression on requests.
   -v, --version                Show application version.