From 92013a0e3fb7216bdf6e5d4713269080c620689a Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Fri, 1 Jul 2016 14:07:46 -0700 Subject: [PATCH 1/8] Move yab to beta status with 0.4.0 release --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11a19047..aedbb22e 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ supports making Thrift requests to both HTTP and TChannel services. **NOTE**: HTTP support is not cross-compatible with Apache Thrift yet, due to missing message envelope support. -`yab` is currently in **alpha** status. +`yab` is currently in **beta** status. ### Installing From cdaf854be6371532dcb57b0096679cf681439dd1 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 7 Jul 2016 09:30:11 -0700 Subject: [PATCH 2/8] benchmark: Round robin peer selection to balance connections Previously, we relied on randomly picking a peer for each connection which could lead to an imbalance. E.g., if there are 3 peers and 3 connections, there was no guarantee that all 3 peers would receive a single connection. Instead of picking randomly, round-robin the peers that can be used for each connection (starting at a random offset) for better load balancing. --- bench_method.go | 21 ++++++++++++-- bench_method_test.go | 67 ++++++++++++++++++++++++++++++++++++++++---- benchmark.go | 1 - server_util_test.go | 9 ++++++ transport.go | 35 +++++++++++++++-------- 5 files changed, 112 insertions(+), 21 deletions(-) diff --git a/bench_method.go b/bench_method.go index 3e4bff27..95871d6c 100644 --- a/bench_method.go +++ b/bench_method.go @@ -21,6 +21,7 @@ package main import ( + "math/rand" "sync" "time" @@ -63,19 +64,35 @@ func (m benchmarkMethod) call(t transport.Transport) (time.Duration, error) { return duration, err } +func hostPortBalancer(hostPorts []string) func(i int) string { + numHostPorts := len(hostPorts) + startOffset := rand.Intn(numHostPorts) + return func(i int) string { + offset := (startOffset + i) % numHostPorts + return hostPorts[offset] + } +} + // WarmTransports returns n transports that have been warmed up. // No requests may fail during the warmup period. func (m benchmarkMethod) WarmTransports(n int, tOpts TransportOptions, warmupRequests int) ([]transport.Transport, error) { + tOpts, err := loadTransportHostPorts(tOpts) + if err != nil { + return nil, err + } + + hostPortFor := hostPortBalancer(tOpts.HostPorts) transports := make([]transport.Transport, n) errs := make([]error, n) var wg sync.WaitGroup for i := range transports { wg.Add(1) - go func(i int) { + go func(i int, tOpts TransportOptions) { defer wg.Done() + tOpts.HostPorts = []string{hostPortFor(i)} transports[i], errs[i] = m.WarmTransport(tOpts, warmupRequests) - }(i) + }(i, tOpts) } wg.Wait() diff --git a/bench_method_test.go b/bench_method_test.go index b1e2c34e..5cf3d4a7 100644 --- a/bench_method_test.go +++ b/bench_method_test.go @@ -22,6 +22,7 @@ package main import ( "fmt" + "math/rand" "sync/atomic" "testing" "time" @@ -168,22 +169,76 @@ func TestBenchmarkMethodCall(t *testing.T) { } } +func TestHostPortBalancer(t *testing.T) { + tests := []struct { + seed int64 + hostPorts []string + want []string + }{ + { + seed: 1, + hostPorts: []string{"1"}, + want: []string{"1", "1", "1"}, + }, + { + seed: 1, + hostPorts: []string{"1", "2"}, + want: []string{"2", "1", "2"}, + }, + { + seed: 2, + hostPorts: []string{"1", "2"}, + want: []string{"1", "2", "1"}, + }, + { + seed: 1, + hostPorts: []string{"1", "2", "3", "4", "5"}, + want: []string{"2", "3", "4"}, + }, + } + + for _, tt := range tests { + rand.Seed(tt.seed) + hostPortFor := hostPortBalancer(tt.hostPorts) + for i, want := range tt.want { + got := hostPortFor(i) + assert.Equal(t, want, got, "hostPortBalancer(%v) seed %v i %v failed", tt.hostPorts, tt.seed, i) + } + } +} + func TestBenchmarkMethodWarmTransportsSuccess(t *testing.T) { + const numServers = 5 m := benchmarkMethodForTest(t, fooMethod, transport.TChannel) - s := newServer(t) - defer s.shutdown() - s.register(fooMethod, methods.echo()) + + counters := make([]*int32, numServers) + servers := make([]*server, numServers) + serverHPs := make([]string, numServers) + for i := range servers { + servers[i] = newServer(t) + defer servers[i].shutdown() + serverHPs[i] = servers[i].hostPort() + + counter, handler := methods.counter() + counters[i] = counter + servers[i].register(fooMethod, handler) + } tOpts := TransportOptions{ ServiceName: "foo", - HostPorts: []string{s.hostPort()}, + HostPorts: serverHPs, } - transports, err := m.WarmTransports(10, tOpts, 1 /* warmupRequests */) + transports, err := m.WarmTransports(numServers, tOpts, 1 /* warmupRequests */) assert.NoError(t, err, "WarmTransports should not fail") - assert.Equal(t, 10, len(transports), "Got unexpected number of transports") + assert.Equal(t, numServers, len(transports), "Got unexpected number of transports") for i, transport := range transports { assert.NotNil(t, transport, "transports[%v] should not be nil", i) } + + // Verify that each server has received one call. + for i, counter := range counters { + assert.EqualValues(t, 1, *counter, "Server %v received unexpected number of calls", i) + } } func TestBenchmarkMethodWarmTransportsError(t *testing.T) { diff --git a/benchmark.go b/benchmark.go index cefaa18f..c2264274 100644 --- a/benchmark.go +++ b/benchmark.go @@ -82,7 +82,6 @@ func runBenchmark(out output, allOpts Options, m benchmarkMethod) { out.Printf(" Max RPS: %v\n", opts.RPS) // Warm up number of connections. - // TODO: If we're making N connections, we should try to select N unique peers connections, err := m.WarmTransports(numConns, allOpts.TOpts, opts.WarmupRequests) if err != nil { out.Fatalf("Failed to warmup connections for benchmark: %v", err) diff --git a/server_util_test.go b/server_util_test.go index bc3c6727..bb24017a 100644 --- a/server_util_test.go +++ b/server_util_test.go @@ -22,6 +22,7 @@ package main import ( "errors" + "sync/atomic" "testing" "github.com/uber/tchannel-go" @@ -110,6 +111,14 @@ func (methodsT) errorIf(f func() bool) handler { } } +func (methodsT) counter() (*int32, handler) { + var count int32 + return &count, methods.errorIf(func() bool { + atomic.AddInt32(&count, 1) + return false + }) +} + func echoServer(t *testing.T, method string, overrideResp []byte) string { s := newServer(t) if overrideResp != nil { diff --git a/transport.go b/transport.go index fa47a8fa..ec02c100 100644 --- a/transport.go +++ b/transport.go @@ -86,31 +86,42 @@ func ensureSameProtocol(hostPorts []string) (string, error) { return lastProtocol, nil } -func getTransport(opts TransportOptions, encoding encoding.Encoding) (transport.Transport, error) { - if opts.ServiceName == "" { - return nil, errServiceRequired - } +func loadTransportHostPorts(opts TransportOptions) (TransportOptions, error) { if len(opts.HostPorts) == 0 && opts.HostPortFile == "" { - return nil, errPeerRequired + return opts, errPeerRequired } hostPorts := opts.HostPorts if opts.HostPortFile != "" { if len(hostPorts) > 0 { - return nil, errPeerOptions + return opts, errPeerOptions } var err error hostPorts, err = parseHostFile(opts.HostPortFile) if err != nil { - return nil, fmt.Errorf("failed to parse host file: %v", err) + return opts, fmt.Errorf("failed to parse host file: %v", err) } if len(hostPorts) == 0 { - return nil, errPeerRequired + return opts, errPeerRequired } } - protocol, err := ensureSameProtocol(hostPorts) + opts.HostPorts = hostPorts + return opts, nil +} + +func getTransport(opts TransportOptions, encoding encoding.Encoding) (transport.Transport, error) { + if opts.ServiceName == "" { + return nil, errServiceRequired + } + + opts, err := loadTransportHostPorts(opts) + if err != nil { + return nil, err + } + + protocol, err := ensureSameProtocol(opts.HostPorts) if err != nil { return nil, err } @@ -124,7 +135,7 @@ func getTransport(opts TransportOptions, encoding encoding.Encoding) (transport. } if protocol == "tchannel" { - remapLocalHost(hostPorts) + remapLocalHost(opts.HostPorts) traceSampleRate := 1.0 if opts.benchmarking { @@ -134,7 +145,7 @@ func getTransport(opts TransportOptions, encoding encoding.Encoding) (transport. topts := transport.TChannelOptions{ SourceService: sourceService, TargetService: opts.ServiceName, - HostPorts: hostPorts, + HostPorts: opts.HostPorts, Encoding: encoding.String(), TransportOpts: opts.TransportOptions, TraceSampleRate: traceSampleRate, @@ -145,7 +156,7 @@ func getTransport(opts TransportOptions, encoding encoding.Encoding) (transport. hopts := transport.HTTPOptions{ SourceService: sourceService, TargetService: opts.ServiceName, - URLs: hostPorts, + URLs: opts.HostPorts, } return transport.NewHTTP(hopts) } From 864ea7d791d45a22d21789e88728568061ffcd53 Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Fri, 15 Jul 2016 08:12:53 -0700 Subject: [PATCH 3/8] Better man page support for pre-formatted code and bullet lists (#63) Now if we start a line with a tab we'll try to re-write it in a more groff-friendly way. --- Makefile | 2 +- doc.go | 67 ++++--- main.go | 30 +++- main_test.go | 67 +++++++ man/yab.1 | 175 +++++++++++++----- man/yab.html | 493 ++++++++++++++++++++++++++++++++++++++------------- 6 files changed, 634 insertions(+), 200 deletions(-) diff --git a/Makefile b/Makefile index 09b218d1..dc1bc4c3 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ install_ci: install update_man: go install . yab --man-page > man/yab.1 - nroff -man man/yab.1 | man2html -title yab > man/yab.html + groff -man -T html man/yab.1 > man/yab.html [[ -d ../yab_ghpages ]] && cp man/yab.html ../yab_ghpages/man.html @echo "Please update gh-pages" diff --git a/doc.go b/doc.go index d254b323..13ec9812 100644 --- a/doc.go +++ b/doc.go @@ -19,7 +19,8 @@ // THE SOFTWARE. // yab is a benchmarking tool for TChannel and HTTP applications. -// It's primarily intended for Thrift applications but supports other encodings like JSON and binary (raw). +// It's primarily intended for Thrift applications but supports other encodings +// like JSON and binary (raw). // // It can be used in a curl-like fashion when benchmarking features are disabled. // @@ -32,70 +33,88 @@ const _reqOptsDesc = `Configures the request data and the encoding. To make Thrift requests, specify a Thrift file and pass the Thrift service and procedure to the method argument (-m or --method) as Service::Method. - $ yab -p localhost:9787 kv -t kv.thrift -m KeyValue::Count -r '{}' + + $ yab -p localhost:9787 kv -t kv.thrift -m KeyValue::Count -r '{}' You can also use positional arguments to specify the method and body: - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Count '{}' + + $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Count '{}' The TChannel health endpoint can be hit without specifying a Thrift file by passing --health. Thrift requests can be specified as JSON or YAML. For example, for a method defined as: - void addUser(1: string name, 2: i32 age) + + void addUser(1: string name, 2: i32 age) You can pass the request as JSON: {"name": "Prashant", age: 100} or as YAML: - name: Prashant - age: 100 + + name: Prashant + age: 100 The request body can be specified on the command line using -r or --request: - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Get -r '{"key": "hello"}' + + $ yab -p localhost:9787 -t kv.thrift kv \ + KeyValue::Get -r '{"key": "hello"}' Or it can be loaded from a file using -f or --file: - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Get --file req.yaml + + $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Get --file req.yaml Binary data can be specified in one of many ways: - - As a string or an array of bytes: "data" or [100, 97, 116, 97] - - As base64: {"base64": "ZGF0YQ=="} - - Loaded from a file: {"file": "data.bin"} + * As a string or an array of bytes: "data" or [100, 97, 116, 97] + * As base64: {"base64": "ZGF0YQ=="} + * Loaded from a file: {"file": "data.bin"} Examples: - $ yab -p localhost:9787 -t kv.thrift kv -m KeyValue::Set -3 '{"key": "hello", "value": [100, 97, 116, 97]}' - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Set -3 '{"key": "hello", "value": {"file": "data.bin"}}' + + $ yab -p localhost:9787 -t kv.thrift kv -m KeyValue::Set \ + -r '{"key": "hello", "value": [100, 97, 116, 97]}' + + $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Set \ + -r '{"key": "hello", "value": {"file": "data.bin"}}' ` const _transportOptsDesc = `Configures the network transport used to make requests. yab can target both TChannel and HTTP endpoints. To specify a TChannel endpoint, specify the peer's host and port: - $ yab -p localhost:9787 [options] + + $ yab -p localhost:9787 [options] + or - $ yab -p tchannel://localhost:9787 [options] + + $ yab -p tchannel://localhost:9787 [options] For HTTP endpoints, specify the URL as the peer, - $ yab -p http://localhost:8080/thrift [options] -The Thrift encoded body will be POSTed to the specified URL. + $ yab -p http://localhost:8080/thrift [options] + +The Thrift-encoded body will be POSTed to the specified URL. Multiple peers can be specified using a peer list using -P or --peer-list. When making a single request, a single peer from this list is selected randomly. -When benchmarking, each connection will randomly select a peer. - $ yab --peer-list hosts.json [options] +When benchmarking, connections will be established in a round-robin fashion, +starting with a random peer. + + $ yab --peer-list hosts.json [options] ` const _benchmarkOptsDesc = `Configures benchmarking, which is disabled by default. -By default, yab will only make a single request. To enable benchmarking, you -must specify the maximum duration for the benchmark by passing -d or --max-duration. +By default, yab will only make a single request. To enable benchmarking, +specify the maximum duration for the benchmark by passing -d or --max-duration. -yab will make requests till either the maximum requests (-n or --max-requests) +yab will make requests until either the maximum requests (-n or --max-requests) or the maximum duration is reached. You can control the rate at which yab makes requests using the --rps flag. An example benchmark command might be: - yab -p localhost:9787 moe --health -n 100000 -d 10s --rps 1000 + + $ yab -p localhost:9787 moe --health -n 100000 -d 10s --rps 1000 This would make requests at 1000 RPS until either the maximum number of requests (100,000) or the maximum duration (10 seconds) is reached. @@ -105,3 +124,5 @@ CPUs on the machine), but will only have one concurrent call per connection. The number of connections and concurrent calls per connection can be controlled using --connections and --concurrency. ` + +/* vim: set tabstop=8:softtabstop=8:shiftwidth=8:noexpandtab */ diff --git a/main.go b/main.go index 14801c5a..8722d73e 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,8 @@ import ( "errors" "log" "os" + "regexp" + "strings" "time" "github.com/yarpc/yab/encoding" @@ -69,20 +71,38 @@ func main() { var errExit = errors.New("sentinel error used to exit cleanly") +func toGroff(s string) string { + // Expand tabbed lines beginning with "-" as items in a bullet list. + s = strings.Replace(s, "\n\t* ", "\n.IP \\[bu]\n", -1 /* all occurences */) + + // Two newlines start a new paragraph. + s = strings.Replace(s, "\n\n", "\n.PP\n", -1) + + // Lines beginning with a tab are interpreted as example code. + // + // See http://liw.fi/manpages/ for an explanation of these + // commands -- tl;dr: turn of paragraph filling and indent the + // block one level. + indentRegexp := regexp.MustCompile(`\t(.*)\n`) + s = indentRegexp.ReplaceAllString(s, ".nf\n.RS\n$1\n.RE\n.fi\n") + + return s +} + func getOptions(args []string, out output) (*Options, error) { opts := newOptions() parser := flags.NewParser(opts, flags.HelpFlag|flags.PassDoubleDash) parser.Usage = "[ ] [OPTIONS]" parser.ShortDescription = "yet another benchmarker" - parser.LongDescription = ` + parser.LongDescription = toGroff(` yab is a benchmarking tool for TChannel and HTTP applications. It's primarily intended for Thrift applications but supports other encodings like JSON and binary (raw). It can be used in a curl-like fashion when benchmarking features are disabled. -` +`) - setGroupDesc(parser, "request", "Request Options", _reqOptsDesc) - setGroupDesc(parser, "transport", "Transport Options", _transportOptsDesc) - setGroupDesc(parser, "benchmark", "Benchmark Options", _benchmarkOptsDesc) + setGroupDesc(parser, "request", "Request Options", toGroff(_reqOptsDesc)) + setGroupDesc(parser, "transport", "Transport Options", toGroff(_transportOptsDesc)) + setGroupDesc(parser, "benchmark", "Benchmark Options", toGroff(_benchmarkOptsDesc)) // If there are no arguments specified, write the help. if len(args) == 0 { diff --git a/main_test.go b/main_test.go index 3d5c9d17..49735876 100644 --- a/main_test.go +++ b/main_test.go @@ -360,3 +360,70 @@ func TestAlises(t *testing.T) { } } } + +func TestToGroff(t *testing.T) { + tests := []struct { + input string + expected string + }{ + { + input: ` +not a bullet list + + * bullet + * list + +not a bullet list`, + expected: ` +not a bullet list +.PP +.IP \[bu] +bullet +.IP \[bu] +list +.PP +not a bullet list`, + }, + + { + input: ` +beginning + + pre-formatted + +middle + + pre-formatted + still pre-formatted + +end`, + expected: ` +beginning +.PP +.nf +.RS +pre-formatted +.RE +.fi +.PP +middle +.PP +.nf +.RS +pre-formatted +.RE +.fi +.nf +.RS +still pre-formatted +.RE +.fi +.PP +end`, + }, + } + + for _, tt := range tests { + assert.Equal(t, toGroff(tt.input), tt.expected) + } +} diff --git a/man/yab.1 b/man/yab.1 index e67d6b44..cd2d91e7 100644 --- a/man/yab.1 +++ b/man/yab.1 @@ -1,4 +1,4 @@ -.TH yab 1 "30 June 2016" +.TH yab 1 "15 July 2016" .SH NAME yab \- yet another benchmarker .SH SYNOPSIS @@ -6,7 +6,7 @@ yab \- yet another benchmarker .SH DESCRIPTION yab is a benchmarking tool for TChannel and HTTP applications. It's primarily intended for Thrift applications but supports other encodings like JSON and binary (raw). - +.PP It can be used in a curl-like fashion when benchmarking features are disabled. .SH OPTIONS @@ -16,41 +16,103 @@ It can be used in a curl-like fashion when benchmarking features are disabled. Displays the application version .SS Request Options Configures the request data and the encoding. - +.PP To make Thrift requests, specify a Thrift file and pass the Thrift service and procedure to the method argument (-m or --method) as Service::Method. - $ yab -p localhost:9787 kv -t kv.thrift -m KeyValue::Count -r '{}' - +.PP +.nf +.RS +$ yab -p localhost:9787 kv -t kv.thrift -m KeyValue::Count -r '{}' +.RE +.fi +.PP You can also use positional arguments to specify the method and body: - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Count '{}' - +.PP +.nf +.RS +$ yab -p localhost:9787 -t kv.thrift kv KeyValue::Count '{}' +.RE +.fi +.PP The TChannel health endpoint can be hit without specifying a Thrift file by passing --health. - +.PP Thrift requests can be specified as JSON or YAML. For example, for a method defined as: - void addUser(1: string name, 2: i32 age) - +.PP +.nf +.RS +void addUser(1: string name, 2: i32 age) +.RE +.fi +.PP You can pass the request as JSON: {"name": "Prashant", age: 100} or as YAML: - name: Prashant - age: 100 - +.PP +.nf +.RS +name: Prashant +.RE +.fi +.nf +.RS +age: 100 +.RE +.fi +.PP The request body can be specified on the command line using -r or --request: - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Get -r '{"key": "hello"}' - +.PP +.nf +.RS +$ yab -p localhost:9787 -t kv.thrift kv \\ +.RE +.fi +.nf +.RS + KeyValue::Get -r '{"key": "hello"}' +.RE +.fi +.PP Or it can be loaded from a file using -f or --file: - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Get --file req.yaml - +.PP +.nf +.RS +$ yab -p localhost:9787 -t kv.thrift kv KeyValue::Get --file req.yaml +.RE +.fi +.PP Binary data can be specified in one of many ways: - - As a string or an array of bytes: "data" or [100, 97, 116, 97] - - As base64: {"base64": "ZGF0YQ=="} - - Loaded from a file: {"file": "data.bin"} - +.IP \\[bu] +As a string or an array of bytes: "data" or [100, 97, 116, 97] +.IP \\[bu] +As base64: {"base64": "ZGF0YQ=="} +.IP \\[bu] +Loaded from a file: {"file": "data.bin"} +.PP Examples: - $ yab -p localhost:9787 -t kv.thrift kv -m KeyValue::Set -3 '{"key": "hello", "value": [100, 97, 116, 97]}' - $ yab -p localhost:9787 -t kv.thrift kv KeyValue::Set -3 '{"key": "hello", "value": {"file": "data.bin"}}' +.PP +.nf +.RS +$ yab -p localhost:9787 -t kv.thrift kv -m KeyValue::Set \\ +.RE +.fi +.nf +.RS + -r '{"key": "hello", "value": [100, 97, 116, 97]}' +.RE +.fi +.PP +.nf +.RS +$ yab -p localhost:9787 -t kv.thrift kv KeyValue::Set \\ +.RE +.fi +.nf +.RS + -r '{"key": "hello", "value": {"file": "data.bin"}}' +.RE +.fi .TP \fB\fB\-e\fR, \fB\-\-encoding\fR\fP @@ -81,22 +143,44 @@ Hit the health endpoint, Meta::health The timeout for each request. E.g., 100ms, 0.5s, 1s. If no unit is specified, milliseconds are assumed. .SS Transport Options Configures the network transport used to make requests. - +.PP yab can target both TChannel and HTTP endpoints. To specify a TChannel endpoint, specify the peer's host and port: - $ yab -p localhost:9787 [options] +.PP +.nf +.RS +$ yab -p localhost:9787 [options] +.RE +.fi +.PP or - $ yab -p tchannel://localhost:9787 [options] - +.PP +.nf +.RS +$ yab -p tchannel://localhost:9787 [options] +.RE +.fi +.PP For HTTP endpoints, specify the URL as the peer, - $ yab -p http://localhost:8080/thrift [options] - -The Thrift encoded body will be POSTed to the specified URL. - +.PP +.nf +.RS +$ yab -p http://localhost:8080/thrift [options] +.RE +.fi +.PP +The Thrift-encoded body will be POSTed to the specified URL. +.PP Multiple peers can be specified using a peer list using -P or --peer-list. When making a single request, a single peer from this list is selected randomly. -When benchmarking, each connection will randomly select a peer. - $ yab --peer-list hosts.json [options] +When benchmarking, connections will be established in a round-robin fashion, +starting with a random peer. +.PP +.nf +.RS +$ yab --peer-list hosts.json [options] +.RE +.fi .TP \fB\fB\-s\fR, \fB\-\-service\fR\fP @@ -115,21 +199,26 @@ Caller will override the default caller name (which is yab-$USER). Custom options for the specific transport being used .SS Benchmark Options Configures benchmarking, which is disabled by default. - -By default, yab will only make a single request. To enable benchmarking, you -must specify the maximum duration for the benchmark by passing -d or --max-duration. - -yab will make requests till either the maximum requests (-n or --max-requests) +.PP +By default, yab will only make a single request. To enable benchmarking, +specify the maximum duration for the benchmark by passing -d or --max-duration. +.PP +yab will make requests until either the maximum requests (-n or --max-requests) or the maximum duration is reached. - +.PP You can control the rate at which yab makes requests using the --rps flag. - +.PP An example benchmark command might be: - yab -p localhost:9787 moe --health -n 100000 -d 10s --rps 1000 - +.PP +.nf +.RS +$ yab -p localhost:9787 moe --health -n 100000 -d 10s --rps 1000 +.RE +.fi +.PP This would make requests at 1000 RPS until either the maximum number of requests (100,000) or the maximum duration (10 seconds) is reached. - +.PP By default, yab will create multiple connections (defaulting to the number of CPUs on the machine), but will only have one concurrent call per connection. The number of connections and concurrent calls per connection can be controlled diff --git a/man/yab.html b/man/yab.html index 95215dbb..0cab507b 100644 --- a/man/yab.html +++ b/man/yab.html @@ -1,178 +1,415 @@ - - -yab - - - -

yab

-
-
-
+
+
+
+
+
+
+
+
+
+yab
 
-
-

SYNOPSIS

-       yab [<service> <method> <body>] [OPTIONS]
+
+
 
+
yab

 
-
-

DESCRIPTION

-       yab  is  a  benchmarking  tool for TChannel and HTTP applications. It’s
-       primarily intended for Thrift applications but supports other encodings
-       like JSON and binary (raw).
+NAME

+SYNOPSIS

+DESCRIPTION

+OPTIONS

 
-       It  can  be  used in a curl‐like fashion when benchmarking features are
-       disabled.
+

 
 
+
+
NAME

 
-
-

OPTIONS

-   Application Options
-       --version
-              Displays the application version
 
-   Request Options
-       Configures the request data and the encoding.
+
yab − yet
+another benchmarker

 
-       To make Thrift requests, specify a Thrift file and pass the Thrift ser‐
-       vice  and  procedure  to  the  method argument (‐m or ‐‐method) as Ser‐
-       vice::Method.
-         $ yab ‐p localhost:9787 kv ‐t kv.thrift ‐m KeyValue::Count ‐r ’{}’
+
+
SYNOPSIS

 
-       You can also use positional arguments to specify the method and body:
-         $ yab ‐p localhost:9787  ‐t kv.thrift kv KeyValue::Count ’{}’
 
-       The TChannel health endpoint can be hit  without  specifying  a  Thrift
-       file by passing ‐‐health.
+
yab
+[<service> <method> <body>] [OPTIONS]

 
-       Thrift  requests  can  be specified as JSON or YAML. For example, for a
-       method defined as:
-         void addUser(1: string name, 2: i32 age)
+
+
DESCRIPTION

 
-       You can pass the request as JSON: {"name": "Prashant", age: 100} or  as
-       YAML:
-         name: Prashant
-         age: 100
 
-       The  request  body  can  be  specified  on the command line using ‐r or
-       ‐‐request:
-         $ yab ‐p localhost:9787 ‐t kv.thrift  kv  KeyValue::Get  ‐r  ’{"key":
-       "hello"}’
+
yab is a
+benchmarking tool for TChannel and HTTP applications.
+It’s primarily intended for Thrift applications but
+supports other encodings like JSON and binary (raw).

 
-       Or it can be loaded from a file using ‐f or ‐‐file:
-         $ yab ‐p localhost:9787 ‐t kv.thrift kv KeyValue::Get ‐‐file req.yaml
+
It can be used
+in a curl-like fashion when benchmarking features are
+disabled.

 
-       Binary data can be specified in one of many ways:
-         ‐ As a string or an array of bytes: "data" or [100, 97, 116, 97]
-         ‐ As base64: {"base64": "ZGF0YQ=="}
-         ‐ Loaded from a file: {"file": "data.bin"}
-              Path of the .thrift file
+
+
OPTIONS

 
-       -m, --method
-              The full Thrift method name (Svc::Method) to invoke
 
-       -r, --request
-              The request body, in JSON or YAML format
+
Application
+Options 

+−−version

 
-       -f, --file
-              Path of a file containing the request body in JSON or YAML
+
Displays the application
+version

 
-       --headers
-              The headers in JSON or YAML format
+
Request
+Options 

+Configures the request data and the encoding.

 
-       --headers-file
-              Path of a file containing the headers in JSON or YAML
+
To make Thrift
+requests, specify a Thrift file and pass the Thrift service
+and procedure to the method argument (-m or --method) as
+Service::Method.

 
-       --health
-              Hit the health endpoint, Meta::health
+
$ yab -p
+localhost:9787 kv -t kv.thrift -m KeyValue::Count -r
+’{}’

 
-       --timeout <default: "1s">
-              The timeout for each request. E.g., 100ms, 0.5s, 1s. If no  unit
-              is specified, milliseconds are assumed.
+
You can also
+use positional arguments to specify the method and body:

 
-   Transport Options
-       Configures the network transport used to make requests.
+
$ yab -p
+localhost:9787 -t kv.thrift kv KeyValue::Count
+’{}’

 
-       yab  can target both TChannel and HTTP endpoints. To specify a TChannel
-       endpoint, specify the peer’s host and port:
-         $ yab ‐p localhost:9787 [options] or
-         $ yab ‐p tchannel://localhost:9787 [options]
+
The TChannel
+health endpoint can be hit without specifying a Thrift file
+by passing --health.

 
-       For HTTP endpoints, specify the URL as the peer,
-         $ yab ‐p http://localhost:8080/thrift [options]
+
Thrift requests
+can be specified as JSON or YAML. For example, for a method
+defined as:

 
-       The Thrift encoded body will be POSTed to the specified URL.
+
void addUser(1:
+string name, 2: i32 age)

 
-       Multiple peers can be specified using a peer list using ‐P  or  ‐‐peer‐
-       list.   When  making  a single request, a single peer from this list is
-       selected randomly.  When benchmarking, each  connection  will  randomly
-       select a peer.
-         $ yab ‐‐peer‐list hosts.json [options]
+
You can pass
+the request as JSON: {"name":
+"Prashant", age: 100} or as YAML:

 
+
name: Prashant
+

+age: 100

 
-       -s, --service
-              The TChannel/Hyperbahn service name
+
The request
+body can be specified on the command line using -r or
+--request:

 
-       -p, --peer
-              The host:port of the service to call
+
$ yab -p
+localhost:9787 -t kv.thrift kv \ 

+KeyValue::Get -r ’{"key":
+"hello"}’

 
-       -P, --peer-list
-              Path of a JSON or YAML file containing a list of host:ports
+
Or it can be
+loaded from a file using -f or --file:

 
-       yab will make requests till either the maximum requests (‐n  or  ‐‐max‐
-       requests) or the maximum duration is reached.
+
$ yab -p
+localhost:9787 -t kv.thrift kv KeyValue::Get --file
+req.yaml

 
-       You  can  control  the rate at which yab makes requests using the ‐‐rps
-       flag.
+
Binary data can
+be specified in one of many ways:

 
-       An example benchmark command might be:
-         yab ‐p localhost:9787 moe ‐‐health ‐n 100000 ‐d 10s ‐‐rps 1000
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

 
-       This would make requests at 1000 RPS until either the maximum number of
-       requests (100,000) or the maximum duration (10 seconds) is reached.
 
-       By  default,  yab  will  create multiple connections (defaulting to the
-       number of CPUs on the machine), but will only have one concurrent  call
-       per  connection.   The  number  of connections and concurrent calls per
-       connection can be controlled using ‐‐connections and ‐‐concurrency.
+
•

 
 
-       -n, --max-requests <default: "1000000">
-              The maximum number of requests to make
+
As a string or an
+array of bytes: "data" or [100, 97, 116, 97]

 
-       -d, --max-duration <default: "0s">
-              The maximum amount of time to run the benchmark for
 
-       --cpus The number of OS threads
+
•

 
-       --connections
-              The number of TCP connections to use
 
-       --warmup <default: "10">
-              The number of requests to make to warmup each connection
+
As base64:
+{"base64": "ZGF0YQ=="}

 
-       --concurrency <default: "1">
-              The number of concurrent calls per connection
 
-       --rps <default: "0">
-              Limit on the number of requests per second. The default  (0)  is
-              no limit.
+
•

 
-       --statsd
-              Optional host:port of a StatsD server to report metrics
 
-   Help Options
-       -h, --help
-              Show this help message
+
Loaded from a file:
+{"file": "data.bin"}

 
+
Examples:

 
+
$ yab -p
+localhost:9787 -t kv.thrift kv -m KeyValue::Set \ 

+-r ’{"key": "hello",
+"value": [100, 97, 116, 97]}’

 
-                                 30 June 2016                           yab(1)
-
-
-
-Man(1) output converted with -man2html -
- - +

$ yab -p +localhost:9787 -t kv.thrift kv KeyValue::Set \
+-r ’{"key": "hello", +"value": {"file": +"data.bin"}}’

+ +

−e, +−−encoding

+ +

The encoding of the data, +options are: Thrift, JSON, raw. Defaults to Thrift if the +method contains ’::’ or a Thrift file is +specified

+ +

−t, +−−thrift

+ +

Path of the .thrift file

+ +

−m, +−−method

+ +

The full Thrift method name +(Svc::Method) to invoke

+ +

−r, +−−request

+ +

The request body, in JSON or +YAML format

+ +

−f, +−−file

+ +

Path of a file containing the +request body in JSON or YAML

+ + +

−−headers

+ +

The headers in JSON or YAML +format

+ + +

−−headers-file

+ +

Path of a file containing the +headers in JSON or YAML

+ + +

−−health

+ +

Hit the health endpoint, +Meta::health

+ +

−−timeout +<default: "1s">

+ +

The timeout for each request. +E.g., 100ms, 0.5s, 1s. If no unit is specified, milliseconds +are assumed.

+ +

Transport +Options
+Configures the network transport used to make requests.

+ +

yab can target +both TChannel and HTTP endpoints. To specify a TChannel +endpoint, specify the peer’s host and port:

+ +

$ yab -p +localhost:9787 [options]

+ +

or

+ +

$ yab -p +tchannel://localhost:9787 [options]

+ +

For HTTP +endpoints, specify the URL as the peer,

+ +

$ yab -p +http://localhost:8080/thrift [options]

+ +

The +Thrift-encoded body will be POSTed to the specified URL.

+ +

Multiple peers +can be specified using a peer list using -P or --peer-list. +When making a single request, a single peer from this list +is selected randomly. When benchmarking, connections will be +established in a round-robin fashion, starting with a random +peer.

+ +

$ yab +--peer-list hosts.json [options]

+ +

−s, +−−service

+ +

The TChannel/Hyperbahn service +name

+ +

−p, +−−peer

+ +

The host:port of the service to +call

+ +

−P, +−−peer-list

+ +

Path of a JSON or YAML file +containing a list of host:ports

+ + +

−−caller

+ +

Caller will override the +default caller name (which is yab-$USER).

+ + + + + + + +
+ + + +

−−topt

 + + +

Custom options for +the specific transport being used

+ +

Benchmark +Options
+Configures benchmarking, which is disabled by default.

+ +

By default, yab +will only make a single request. To enable benchmarking, +specify the maximum duration for the benchmark by passing -d +or --max-duration.

+ +

yab will make +requests until either the maximum requests (-n or +--max-requests) or the maximum duration is reached.

+ +

You can control +the rate at which yab makes requests using the --rps +flag.

+ +

An example +benchmark command might be:

+ +

$ yab -p +localhost:9787 moe --health -n 100000 -d 10s --rps 1000

+ +

This would make +requests at 1000 RPS until either the maximum number of +requests (100,000) or the maximum duration (10 seconds) is +reached.

+ +

By default, yab +will create multiple connections (defaulting to the number +of CPUs on the machine), but will only have one concurrent +call per connection. The number of connections and +concurrent calls per connection can be controlled using +--connections and --concurrency.
+−n, −−max-requests <default: +"1000000">

+ +

The maximum number of requests +to make

+ +

−d, +−−max-duration <default: +"0s">

+ +

The maximum amount of time to +run the benchmark for

+ + + + + + + + +
+ + + +

−−cpus

 + + +

The number of OS +threads

 +
+ + +

−−connections

+ +

The number of TCP connections +to use

+ +

−−warmup +<default: "10">

+ +

The number of requests to make +to warmup each connection

+ + +

−−concurrency +<default: "1">

+ +

The number of concurrent calls +per connection

+ +

−−rps +<default: "0">

+ +

Limit on the number of requests +per second. The default (0) is no limit.

+ + +

−−statsd

+ +

Optional host:port of a StatsD +server to report metrics

+ +

Help Options +
+−h, −−help

+ +

Show this help message

+
+ + From f813a61511b0018b3e1f77db80a0b3ecf0066d87 Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Fri, 15 Jul 2016 17:11:22 -0700 Subject: [PATCH 4/8] Read defaults from ~/.config/yab/defaults.ini (#62) --- glide.lock | 6 ++-- glide.yaml | 1 + main.go | 45 +++++++++++++++++++++++++-- main_test.go | 37 ++++++++++++++++++++++ testdata/ini/invalid/yab/defaults.ini | 2 ++ testdata/ini/valid/yab/defaults.ini | 2 ++ 6 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 testdata/ini/invalid/yab/defaults.ini create mode 100644 testdata/ini/valid/yab/defaults.ini diff --git a/glide.lock b/glide.lock index bfff651a..d7673ecd 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: fd05ccae439afc6f74a6c576c98a279826a6111f91bf36db475bccf3bbd3f9eb -updated: 2016-06-01T08:45:45.209063968-07:00 +hash: 0badca56130abf1fc7d731d7e6b3adc1e974185c6d63431ac32ec36c7671e2a6 +updated: 2016-07-08T16:43:42.1722035-07:00 imports: - name: github.com/apache/thrift version: 39a09ac5e49481d39dd1bcb6757ffe182e3df20a @@ -9,6 +9,8 @@ imports: version: 725f76bd0300c1f9562e76fd94c27aa17e41243e subpackages: - statsd +- name: github.com/casimir/xdg-go + version: 372ccc2180dab73316615641d5617c5ed3e35529 - name: github.com/davecgh/go-spew version: 5215b55f46b2b919f50a1df0eaa5886afe4e3b3d subpackages: diff --git a/glide.yaml b/glide.yaml index f1289547..33984a72 100644 --- a/glide.yaml +++ b/glide.yaml @@ -28,3 +28,4 @@ import: version: master subpackages: - lib/go/thrift +- package: github.com/casimir/xdg-go diff --git a/main.go b/main.go index 8722d73e..32720f2b 100644 --- a/main.go +++ b/main.go @@ -23,12 +23,14 @@ package main import ( "encoding/json" "errors" + "fmt" "log" "os" "regexp" "strings" "time" + "github.com/casimir/xdg-go" "github.com/yarpc/yab/encoding" "github.com/yarpc/yab/transport" @@ -89,15 +91,30 @@ func toGroff(s string) string { return s } -func getOptions(args []string, out output) (*Options, error) { +func newParser() (*flags.Parser, *Options) { opts := newOptions() - parser := flags.NewParser(opts, flags.HelpFlag|flags.PassDoubleDash) + return flags.NewParser(opts, flags.HelpFlag|flags.PassDoubleDash), opts +} + +func getOptions(args []string, out output) (*Options, error) { + parser, opts := newParser() parser.Usage = "[ ] [OPTIONS]" parser.ShortDescription = "yet another benchmarker" parser.LongDescription = toGroff(` yab is a benchmarking tool for TChannel and HTTP applications. It's primarily intended for Thrift applications but supports other encodings like JSON and binary (raw). It can be used in a curl-like fashion when benchmarking features are disabled. + +Default options can be specified in a ~/.config/yab/defaults.ini file with contents similar to this: + + [request] + timeout = 2s + + [transport] + peer-list = "/path/to/peer/list.json" + + [benchmark] + warmup = 10 `) setGroupDesc(parser, "request", "Request Options", toGroff(_reqOptsDesc)) @@ -110,6 +127,11 @@ It can be used in a curl-like fashion when benchmarking features are disabled. return opts, errExit } + // Read defaults if they're available. + if err := parseDefaultConfigs(parser); err != nil { + return nil, fmt.Errorf("error reading defaults: %v\n", err) + } + remaining, err := parser.ParseArgs(args) if err != nil { if ferr, ok := err.(*flags.Error); ok { @@ -159,6 +181,25 @@ func parseAndRun(out output) { runWithOptions(*opts, out) } +// parseDefaultConfigs reads defaults from ~/.config/yab/defaults.ini if they're +// available. +func parseDefaultConfigs(parser *flags.Parser) error { + app := xdg.App{Name: "yab"} + + configFile := app.ConfigPath("defaults.ini") + if _, err := os.Stat(configFile); err != nil { + // No defaults file to read + return nil + } + + iniParser := flags.NewIniParser(parser) + if err := iniParser.ParseFile(configFile); err != nil { + return fmt.Errorf("couldn't read %v: %v", configFile, err) + } + + return nil +} + func runWithOptions(opts Options, out output) { reqInput, err := getRequestInput(opts.ROpts.RequestJSON, opts.ROpts.RequestFile) if err != nil { diff --git a/main_test.go b/main_test.go index 49735876..b8bbe16c 100644 --- a/main_test.go +++ b/main_test.go @@ -24,6 +24,7 @@ import ( "bytes" "fmt" "os" + "path" "testing" "time" @@ -361,6 +362,42 @@ func TestAlises(t *testing.T) { } } +func TestParseIniFile(t *testing.T) { + configHomeEnv := "XDG_CONFIG_HOME" + originalConfigHome := os.Getenv(configHomeEnv) + defer os.Setenv(configHomeEnv, originalConfigHome) + + tests := []struct { + message string + configPath string + expectedError string + }{ + { + message: "valid ini file should parse correctly", + configPath: "valid", + }, + { + message: "absent ini file should be ignored", + configPath: "missing", + }, + { + message: "invalid ini file should raise error", + configPath: "invalid", + expectedError: "couldn't read testdata/ini/invalid/yab/defaults.ini: testdata/ini/invalid/yab/defaults.ini:2: time: unknown unit foo in duration 3foo", + }, + } + for _, tt := range tests { + os.Setenv(configHomeEnv, path.Join("testdata", "ini", tt.configPath)) + parser, _ := newParser() + err := parseDefaultConfigs(parser) + if tt.expectedError == "" { + assert.NoError(t, err, tt.message) + } else { + assert.EqualError(t, err, tt.expectedError, tt.message) + } + } +} + func TestToGroff(t *testing.T) { tests := []struct { input string diff --git a/testdata/ini/invalid/yab/defaults.ini b/testdata/ini/invalid/yab/defaults.ini new file mode 100644 index 00000000..cf0a992b --- /dev/null +++ b/testdata/ini/invalid/yab/defaults.ini @@ -0,0 +1,2 @@ +[request] +timeout = 3foo diff --git a/testdata/ini/valid/yab/defaults.ini b/testdata/ini/valid/yab/defaults.ini new file mode 100644 index 00000000..2b2d6d5d --- /dev/null +++ b/testdata/ini/valid/yab/defaults.ini @@ -0,0 +1,2 @@ +[request] +timeout = 2s From d16ef4bd1f3e81d9628b4ec6f9e82975702a66fa Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Fri, 15 Jul 2016 17:15:00 -0700 Subject: [PATCH 5/8] Update man page for XDG config info (#64) --- man/yab.1 | 35 +++++++++++++++++++++++++++++++++++ man/yab.html | 17 ++++++++++++++++- 2 files changed, 51 insertions(+), 1 deletion(-) diff --git a/man/yab.1 b/man/yab.1 index cd2d91e7..2db17e72 100644 --- a/man/yab.1 +++ b/man/yab.1 @@ -8,6 +8,41 @@ yab \- yet another benchmarker yab is a benchmarking tool for TChannel and HTTP applications. It's primarily intended for Thrift applications but supports other encodings like JSON and binary (raw). .PP It can be used in a curl-like fashion when benchmarking features are disabled. +.PP +Default options can be specified in a ~/.config/yab/defaults.ini file with contents similar to this: +.PP +.nf +.RS +[request] +.RE +.fi +.nf +.RS +timeout = 2s +.RE +.fi +.PP +.nf +.RS +[transport] +.RE +.fi +.nf +.RS +peer-list = "/path/to/peer/list.json" +.RE +.fi +.PP +.nf +.RS +[benchmark] +.RE +.fi +.nf +.RS +warmup = 10 +.RE +.fi .SH OPTIONS .SS Application Options diff --git a/man/yab.html b/man/yab.html index 0cab507b..8eb25334 100644 --- a/man/yab.html +++ b/man/yab.html @@ -1,5 +1,5 @@ - + @@ -54,6 +54,21 @@

DESCRIPTION

in a curl-like fashion when benchmarking features are disabled.

+

Default options +can be specified in a ~/.config/yab/defaults.ini file with +contents similar to this:

+ +

[request]
+timeout = 2s

+ +

[transport] +
+peer-list = "/path/to/peer/list.json"

+ +

[benchmark] +
+warmup = 10

+

OPTIONS

From 495e00c7cd1d54aa01f63a24f9ae32c37988b203 Mon Sep 17 00:00:00 2001 From: Bryce Lampe Date: Mon, 18 Jul 2016 11:39:45 -0700 Subject: [PATCH 6/8] Don't leak groff to help output (#66) --- main.go | 12 ++++++------ main_test.go | 3 +++ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/main.go b/main.go index 32720f2b..bdf9c596 100644 --- a/main.go +++ b/main.go @@ -100,7 +100,7 @@ func getOptions(args []string, out output) (*Options, error) { parser, opts := newParser() parser.Usage = "[ ] [OPTIONS]" parser.ShortDescription = "yet another benchmarker" - parser.LongDescription = toGroff(` + parser.LongDescription = ` yab is a benchmarking tool for TChannel and HTTP applications. It's primarily intended for Thrift applications but supports other encodings like JSON and binary (raw). It can be used in a curl-like fashion when benchmarking features are disabled. @@ -115,11 +115,7 @@ Default options can be specified in a ~/.config/yab/defaults.ini file with conte [benchmark] warmup = 10 -`) - - setGroupDesc(parser, "request", "Request Options", toGroff(_reqOptsDesc)) - setGroupDesc(parser, "transport", "Transport Options", toGroff(_transportOptsDesc)) - setGroupDesc(parser, "benchmark", "Benchmark Options", toGroff(_benchmarkOptsDesc)) +` // If there are no arguments specified, write the help. if len(args) == 0 { @@ -150,6 +146,10 @@ Default options can be specified in a ~/.config/yab/defaults.ini file with conte } if opts.ManPage { + parser.LongDescription = toGroff(parser.LongDescription) + setGroupDesc(parser, "request", "Request Options", toGroff(_reqOptsDesc)) + setGroupDesc(parser, "transport", "Transport Options", toGroff(_transportOptsDesc)) + setGroupDesc(parser, "benchmark", "Benchmark Options", toGroff(_benchmarkOptsDesc)) parser.WriteManPage(os.Stdout) return opts, errExit } diff --git a/main_test.go b/main_test.go index b8bbe16c..2c5efcea 100644 --- a/main_test.go +++ b/main_test.go @@ -226,6 +226,9 @@ func TestHelpOutput(t *testing.T) { buf, out := getOutput(t) parseAndRun(out) assert.Contains(t, buf.String(), "Usage:", "Expected help output") + + // Make sure we didn't leak any groff from the man-page output. + assert.NotContains(t, buf.String(), ".PP") } } From b9640b57a64e3a75e6d1a375ed11a3893285d2a2 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Mon, 18 Jul 2016 13:23:43 -0700 Subject: [PATCH 7/8] Add option to disable enveloping (#65) Some HTTP services don't use Thrift envelopes, so add option to disable envelopes. --- bench_method_test.go | 2 +- main.go | 8 +++--- main_test.go | 64 ++++++++++++++++++++++++++++++++++++++++++++ options.go | 19 ++++++------- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/bench_method_test.go b/bench_method_test.go index 5cf3d4a7..6992128a 100644 --- a/bench_method_test.go +++ b/bench_method_test.go @@ -44,7 +44,7 @@ func benchmarkMethodForTest(t *testing.T, methodString string, p transport.Proto serializer, err := NewSerializer(rOpts) require.NoError(t, err, "Failed to create Thrift serializer") - serializer = withTransportSerializer(p, serializer) + serializer = withTransportSerializer(p, serializer, rOpts) req, err := serializer.Request(nil) require.NoError(t, err, "Failed to serialize Thrift body") diff --git a/main.go b/main.go index bdf9c596..4565aba9 100644 --- a/main.go +++ b/main.go @@ -222,7 +222,7 @@ func runWithOptions(opts Options, out output) { out.Fatalf("Failed while parsing options: %v\n", err) } - serializer = withTransportSerializer(transport.Protocol(), serializer) + serializer = withTransportSerializer(transport.Protocol(), serializer, opts.ROpts) // req is the transport.Request that will be used to make a call. req, err := serializer.Request(reqInput) @@ -275,8 +275,10 @@ type noEnveloper interface { // withTransportSerializer may modify the serializer for the transport used. // E.g. Thrift payloads are not enveloped when used with TChannel. -func withTransportSerializer(p transport.Protocol, s encoding.Serializer) encoding.Serializer { - if p == transport.TChannel && s.Encoding() == encoding.Thrift { +func withTransportSerializer(p transport.Protocol, s encoding.Serializer, rOpts RequestOptions) encoding.Serializer { + switch { + case p == transport.TChannel && s.Encoding() == encoding.Thrift, + rOpts.DisableThriftEnvelopes: s = s.(noEnveloper).WithoutEnvelopes() } return s diff --git a/main_test.go b/main_test.go index 2c5efcea..7ac1e297 100644 --- a/main_test.go +++ b/main_test.go @@ -29,9 +29,12 @@ import ( "time" "github.com/yarpc/yab/encoding" + "github.com/yarpc/yab/transport" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/thriftrw/thriftrw-go/protocol" + "github.com/thriftrw/thriftrw-go/wire" "github.com/uber/tchannel-go/testutils" "github.com/uber/tchannel-go/thrift" ) @@ -467,3 +470,64 @@ end`, assert.Equal(t, toGroff(tt.input), tt.expected) } } + +func TestWithTransportSerializer(t *testing.T) { + validRequestOpts := RequestOptions{ + ThriftFile: validThrift, + MethodName: fooMethod, + } + noEnvelopeOpts := validRequestOpts + noEnvelopeOpts.DisableThriftEnvelopes = true + + tests := []struct { + protocol transport.Protocol + rOpts RequestOptions + want []byte + }{ + { + protocol: transport.HTTP, + rOpts: validRequestOpts, + want: encodeEnveloped(wire.Envelope{ + Name: "foo", + Type: wire.Call, + Value: wire.NewValueStruct(wire.Struct{}), + }), + }, + { + protocol: transport.HTTP, + rOpts: noEnvelopeOpts, + want: []byte{0}, + }, + { + protocol: transport.TChannel, + rOpts: validRequestOpts, + want: []byte{0}, + }, + { + protocol: transport.TChannel, + rOpts: noEnvelopeOpts, + want: []byte{0}, + }, + } + + for _, tt := range tests { + serializer, err := NewSerializer(tt.rOpts) + require.NoError(t, err, "Failed to create serializer for %+v", tt.rOpts) + + serializer = withTransportSerializer(tt.protocol, serializer, tt.rOpts) + req, err := serializer.Request(nil) + if !assert.NoError(t, err, "Failed to serialize request for %+v", tt.rOpts) { + continue + } + + assert.Equal(t, tt.want, req.Body, "Body mismatch for %+v", tt.rOpts) + } +} + +func encodeEnveloped(e wire.Envelope) []byte { + buf := &bytes.Buffer{} + if err := protocol.Binary.EncodeEnveloped(e, buf); err != nil { + panic(fmt.Errorf("Binary.EncodeEnveloped(%v) failed: %v", e, err)) + } + return buf.Bytes() +} diff --git a/options.go b/options.go index a82aecbe..06e58e2f 100644 --- a/options.go +++ b/options.go @@ -39,15 +39,16 @@ type Options struct { // RequestOptions are request related options type RequestOptions struct { - Encoding encoding.Encoding `short:"e" long:"encoding" description:"The encoding of the data, options are: Thrift, JSON, raw. Defaults to Thrift if the method contains '::' or a Thrift file is specified"` - ThriftFile string `short:"t" long:"thrift" description:"Path of the .thrift file"` - MethodName string `short:"m" long:"method" description:"The full Thrift method name (Svc::Method) to invoke"` - RequestJSON string `short:"r" long:"request" description:"The request body, in JSON or YAML format"` - RequestFile string `short:"f" long:"file" description:"Path of a file containing the request body in JSON or YAML"` - HeadersJSON string `long:"headers" description:"The headers in JSON or YAML format"` - HeadersFile string `long:"headers-file" description:"Path of a file containing the headers in JSON or YAML"` - Health bool `long:"health" description:"Hit the health endpoint, Meta::health"` - Timeout timeMillisFlag `long:"timeout" default:"1s" description:"The timeout for each request. E.g., 100ms, 0.5s, 1s. If no unit is specified, milliseconds are assumed."` + Encoding encoding.Encoding `short:"e" long:"encoding" description:"The encoding of the data, options are: Thrift, JSON, raw. Defaults to Thrift if the method contains '::' or a Thrift file is specified"` + ThriftFile string `short:"t" long:"thrift" description:"Path of the .thrift file"` + MethodName string `short:"m" long:"method" description:"The full Thrift method name (Svc::Method) to invoke"` + RequestJSON string `short:"r" long:"request" description:"The request body, in JSON or YAML format"` + RequestFile string `short:"f" long:"file" description:"Path of a file containing the request body in JSON or YAML"` + HeadersJSON string `long:"headers" description:"The headers in JSON or YAML format"` + HeadersFile string `long:"headers-file" description:"Path of a file containing the headers in JSON or YAML"` + Health bool `long:"health" description:"Hit the health endpoint, Meta::health"` + Timeout timeMillisFlag `long:"timeout" default:"1s" description:"The timeout for each request. E.g., 100ms, 0.5s, 1s. If no unit is specified, milliseconds are assumed."` + DisableThriftEnvelopes bool `long:"disable-thrift-envelope" description:"Disables Thrift envelopes (disabled by default for TChannel)"` // These are aliases for tcurl compatibility. Aliases struct { From ae84f5cdf76d570b16257f1ca05f5b5888ebab95 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Mon, 18 Jul 2016 13:24:38 -0700 Subject: [PATCH 8/8] Bump version to 0.5.0 --- version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.go b/version.go index 66939bd0..aae65b31 100644 --- a/version.go +++ b/version.go @@ -22,4 +22,4 @@ package main // versionString is the sem-ver version string for yab. // It will be bumped explicitly on releases. -var versionString = "0.4.0" +var versionString = "0.5.0"