From 8d4f463e9e5326d6b86010a8b9d7f73533ca6d4c Mon Sep 17 00:00:00 2001 From: Bojan Date: Mon, 16 Apr 2018 20:27:11 -0300 Subject: [PATCH] Add protoset support. Fixes #5. (#10) Add protoset support. Fixes #5. (#10) --- README.md | 18 ++++++++ cmd/grpcannon/main.go | 25 +++++++--- config/config.go | 24 ++++++---- config/config_test.go | 8 ++-- data_test.go | 4 +- protodesc/protodesc.go | 84 ++++++++++++++++++++++++++++++++-- protodesc/protodesc_test.go | 58 ++++++++++++++++++++--- requester_test.go | 10 ++-- testdata/bundle.protoset | Bin 0 -> 512 bytes testdata/bundle/cap.proto | 9 ++++ testdata/bundle/common.proto | 14 ++++++ testdata/bundle/greeter.proto | 12 +++++ 12 files changed, 228 insertions(+), 38 deletions(-) create mode 100644 testdata/bundle.protoset create mode 100644 testdata/bundle/cap.proto create mode 100644 testdata/bundle/common.proto create mode 100644 testdata/bundle/greeter.proto diff --git a/README.md b/README.md index cbfe2d2f..88f9800c 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,10 @@ Download a prebuilt executable binary from the [releases page](https://github.co Usage: grpcannon [options...] Options: -proto The protocol buffer file. + -protoset The compiled protoset file. Alternative to proto. -proto takes precedence. -call A fully-qualified method name in 'service/method' or 'service.method' format. -cert The file containing the CA root cert file. + -cname an override of the expect Server Cname presented by the server. -c Number of requests to run concurrently. Total number of requests cannot be smaller than the concurrency level. Default is 50. @@ -81,6 +83,22 @@ grpcannon -proto ./greeter.proto -call helloworld.Greeter.SayHelloCS -d '[{"name If a single object is given for data it is sent as every message. +We can also use `.protoset` files which can bundle multiple protoco buffer files into one binary file. + +Create a protoset + +``` +protoc --proto_path=. --descriptor_set_out=bundle.protoset *.proto +``` + +And then use it as input to `grpcannon` with `-protoset` option: + +``` +./grpcannon -protoset ./bundle.protoset -call helloworld.Greeter.SayHello -d '{"name":"Bob"}' -n 1000 -c 10 0.0.0.0:50051 +``` + +Note that only one of `-proto` or `-protoset` options will be used. `-proto` takes precedence. + Example `grpcannon.json` ```json diff --git a/cmd/grpcannon/main.go b/cmd/grpcannon/main.go index 3e13214c..b453c120 100644 --- a/cmd/grpcannon/main.go +++ b/cmd/grpcannon/main.go @@ -13,16 +13,18 @@ import ( "github.com/bojand/grpcannon/config" "github.com/bojand/grpcannon/printer" "github.com/bojand/grpcannon/protodesc" + "github.com/jhump/protoreflect/desc" ) var ( // set by goreleaser with -ldflags="-X main.version=..." version = "dev" - proto = flag.String("proto", "", `The .proto file.`) - call = flag.String("call", "", `A fully-qualified symbol name.`) - cert = flag.String("cert", "", "Client certificate file. If Omitted insecure is used.") - cname = flag.String("cname", "", "Server Cert CName Override - useful for self signed certs") + proto = flag.String("proto", "", `The .proto file.`) + protoset = flag.String("protoset", "", `The .protoset file.`) + call = flag.String("call", "", `A fully-qualified symbol name.`) + cert = flag.String("cert", "", "Client certificate file. If Omitted insecure is used.") + cname = flag.String("cname", "", "Server Cert CName Override - useful for self signed certs") c = flag.Int("c", 50, "Number of requests to run concurrently.") n = flag.Int("n", 200, "Number of requests to run. Default is 200.") @@ -53,6 +55,7 @@ var ( var usage = `Usage: grpcannon [options...] Options: -proto The protocol buffer file. + -protoset The compiled protoset file. Alternative to proto. -proto takes precedence. -call A fully-qualified method name in 'service/method' or 'service.method' format. -cert The file containing the CA root cert file. -cname an override of the expect Server Cname presented by the server. @@ -119,7 +122,7 @@ func main() { iPaths = strings.Split(pathsTrimmed, ",") } - cfg, err = config.New(*proto, *call, *cert, *cname, *n, *c, *q, *z, *t, + cfg, err = config.New(*proto, *protoset, *call, *cert, *cname, *n, *c, *q, *z, *t, *data, *dataPath, *md, *mdPath, *output, *format, host, *ct, *kt, *cpus, iPaths) if err != nil { errAndExit(err.Error()) @@ -157,7 +160,7 @@ func usageAndExit(msg string) { } func runTest(config *config.Config) (*grpcannon.Report, error) { - mtd, err := protodesc.GetMethodDesc(config.Call, config.Proto, config.ImportPaths) + mtd, err := getMethodDesc(config) if err != nil { return nil, err } @@ -165,7 +168,7 @@ func runTest(config *config.Config) (*grpcannon.Report, error) { opts := &grpcannon.Options{ Host: config.Host, Cert: config.Cert, - CName: config.CName, + CName: config.CName, N: config.N, C: config.C, QPS: config.QPS, @@ -199,3 +202,11 @@ func runTest(config *config.Config) (*grpcannon.Report, error) { return reqr.Run() } + +func getMethodDesc(config *config.Config) (*desc.MethodDescriptor, error) { + if config.Proto != "" { + return protodesc.GetMethodDescFromProto(config.Call, config.Proto, config.ImportPaths) + } else { + return protodesc.GetMethodDescFromProtoSet(config.Call, config.Protoset) + } +} diff --git a/config/config.go b/config/config.go index 1b4e7291..c7032984 100644 --- a/config/config.go +++ b/config/config.go @@ -15,9 +15,10 @@ import ( // Config for the run. type Config struct { Proto string `json:"proto"` + Protoset string `json:"protoset"` Call string `json:"call"` Cert string `json:"cert"` - CName string `json:"cName"` + CName string `json:"cName"` N int `json:"n"` C int `json:"c"` QPS int `json:"q"` @@ -36,16 +37,17 @@ type Config struct { ImportPaths []string `json:"i,omitempty"` } -// NewConfig creates a new config -func New(proto, call, cert, cName string, n, c, qps int, z time.Duration, timeout int, +// New creates a new config +func New(proto, protoset, call, cert, cName string, n, c, qps int, z time.Duration, timeout int, data, dataPath, metadata, mdPath, output, format, host string, dialTimout, keepaliveTime, cpus int, importPaths []string) (*Config, error) { cfg := &Config{ Proto: proto, + Protoset: protoset, Call: call, Cert: cert, - CName: cName, + CName: cName, N: n, C: c, QPS: qps, @@ -110,12 +112,18 @@ func (c *Config) Default() { // Validate the config func (c *Config) Validate() error { - if err := requiredString(c.Proto); err != nil { - return errors.Wrap(err, "proto") + if strings.TrimSpace(c.Proto) == "" && strings.TrimSpace(c.Protoset) == "" { + return errors.New("Proto or Protoset required") } - if filepath.Ext(c.Proto) != ".proto" { - return errors.Errorf(fmt.Sprintf("proto: must have .proto extension")) + if strings.TrimSpace(c.Proto) != "" { + if filepath.Ext(c.Proto) != ".proto" { + return errors.Errorf(fmt.Sprintf("proto: must have .proto extension")) + } + } else { + if filepath.Ext(c.Protoset) != ".protoset" { + return errors.Errorf(fmt.Sprintf("protoset: must have .protoset extension")) + } } if err := requiredString(c.Call); err != nil { diff --git a/config/config_test.go b/config/config_test.go index 5f666efb..3ef39b5a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" ) -const expected = `{"proto":"asdf","call":"","cert":"","cName":"","n":0,"c":0,"q":0,"t":0,"D":"","M":"","o":"","O":"oval","host":"","T":0,"L":0,"cpus":0,"z":"4h30m0s"}` +const expected = `{"proto":"asdf","protoset":"","call":"","cert":"","cName":"","n":0,"c":0,"q":0,"t":0,"D":"","M":"","o":"","O":"oval","host":"","T":0,"L":0,"cpus":0,"z":"4h30m0s"}` func TestConfig_MarshalJSON(t *testing.T) { z, _ := time.ParseDuration("4h30m") @@ -260,7 +260,7 @@ func TestConfig_ReadConfig(t *testing.T) { Call: "mycall", Data: data, Cert: "mycert", - CName: "localhost", + CName: "localhost", N: 200, C: 50, QPS: 0, @@ -299,7 +299,7 @@ func TestConfig_ReadConfig(t *testing.T) { Call: "mycall", Data: data, Cert: "mycert", - CName: "localhost", + CName: "localhost", N: 200, C: 50, QPS: 0, @@ -331,7 +331,7 @@ func TestConfig_Validate(t *testing.T) { t.Run("missing proto", func(t *testing.T) { c := &Config{} err := c.Validate() - assert.Equal(t, "proto: is required", err.Error()) + assert.Equal(t, "Proto or Protoset required", err.Error()) }) t.Run("invalid proto", func(t *testing.T) { diff --git a/data_test.go b/data_test.go index b67e4738..ba2c9516 100644 --- a/data_test.go +++ b/data_test.go @@ -159,7 +159,7 @@ func TestData_isMapData(t *testing.T) { } func TestData_createPayloads(t *testing.T) { - mtdUnary, err := protodesc.GetMethodDesc( + mtdUnary, err := protodesc.GetMethodDescFromProto( "helloworld.Greeter.SayHello", "./testdata/greeter.proto", nil) @@ -167,7 +167,7 @@ func TestData_createPayloads(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, mtdUnary) - mtdClientStreaming, err := protodesc.GetMethodDesc( + mtdClientStreaming, err := protodesc.GetMethodDescFromProto( "helloworld.Greeter.SayHelloCS", "./testdata/greeter.proto", nil) diff --git a/protodesc/protodesc.go b/protodesc/protodesc.go index 45bf7688..f237d606 100644 --- a/protodesc/protodesc.go +++ b/protodesc/protodesc.go @@ -2,16 +2,19 @@ package protodesc import ( "fmt" + "io/ioutil" "path/filepath" "strings" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/protoc-gen-go/descriptor" "github.com/jhump/protoreflect/desc" "github.com/jhump/protoreflect/desc/protoparse" ) -// GetMethodDesc gets method descitor for the given call symbol in the file given my protoPath +// GetMethodDescFromProto gets method descritor for the given call symbol from proto file given my path proto // imports is used for import paths in parsing the proto file -func GetMethodDesc(call, proto string, imports []string) (*desc.MethodDescriptor, error) { +func GetMethodDescFromProto(call, proto string, imports []string) (*desc.MethodDescriptor, error) { p := &protoparse.Parser{ImportPaths: imports} filename := proto @@ -26,19 +29,57 @@ func GetMethodDesc(call, proto string, imports []string) (*desc.MethodDescriptor fileDesc := fds[0] + files := map[string]*desc.FileDescriptor{} + files[fileDesc.GetName()] = fileDesc + + return getMethodDesc(call, files) +} + +// GetMethodDescFromProtoSet gets method descritor for the given call symbol from protoset file given my path protoset +func GetMethodDescFromProtoSet(call, protoset string) (*desc.MethodDescriptor, error) { + b, err := ioutil.ReadFile(protoset) + if err != nil { + return nil, fmt.Errorf("could not load protoset file %q: %v", protoset, err) + } + + var fds descriptor.FileDescriptorSet + err = proto.Unmarshal(b, &fds) + if err != nil { + return nil, fmt.Errorf("could not parse contents of protoset file %q: %v", protoset, err) + } + + unresolved := map[string]*descriptor.FileDescriptorProto{} + for _, fd := range fds.File { + unresolved[fd.GetName()] = fd + } + resolved := map[string]*desc.FileDescriptor{} + for _, fd := range fds.File { + _, err := resolveFileDescriptor(unresolved, resolved, fd.GetName()) + if err != nil { + return nil, err + } + } + + return getMethodDesc(call, resolved) +} + +func getMethodDesc(call string, files map[string]*desc.FileDescriptor) (*desc.MethodDescriptor, error) { svc, mth := parseSymbol(call) if svc == "" || mth == "" { return nil, fmt.Errorf("given method name %q is not in expected format: 'service/method' or 'service.method'", call) } - dsc := fileDesc.FindSymbol(svc) + dsc, err := findServiceSymbol(files, svc) + if err != nil { + return nil, err + } if dsc == nil { - return nil, fmt.Errorf("target server does not expose service %q", svc) + return nil, fmt.Errorf("cannot find service %q", svc) } sd, ok := dsc.(*desc.ServiceDescriptor) if !ok { - return nil, fmt.Errorf("target server does not expose service %q", svc) + return nil, fmt.Errorf("cannot find service %q", svc) } mtd := sd.FindMethodByName(mth) @@ -49,6 +90,39 @@ func GetMethodDesc(call, proto string, imports []string) (*desc.MethodDescriptor return mtd, nil } +func resolveFileDescriptor(unresolved map[string]*descriptor.FileDescriptorProto, resolved map[string]*desc.FileDescriptor, filename string) (*desc.FileDescriptor, error) { + if r, ok := resolved[filename]; ok { + return r, nil + } + fd, ok := unresolved[filename] + if !ok { + return nil, fmt.Errorf("no descriptor found for %q", filename) + } + deps := make([]*desc.FileDescriptor, 0, len(fd.GetDependency())) + for _, dep := range fd.GetDependency() { + depFd, err := resolveFileDescriptor(unresolved, resolved, dep) + if err != nil { + return nil, err + } + deps = append(deps, depFd) + } + result, err := desc.CreateFileDescriptor(fd, deps...) + if err != nil { + return nil, err + } + resolved[filename] = result + return result, nil +} + +func findServiceSymbol(resolved map[string]*desc.FileDescriptor, fullyQualifiedName string) (desc.Descriptor, error) { + for _, fd := range resolved { + if dsc := fd.FindSymbol(fullyQualifiedName); dsc != nil { + return dsc, nil + } + } + return nil, fmt.Errorf("cannot find service %q", fullyQualifiedName) +} + func parseSymbol(svcAndMethod string) (string, string) { pos := strings.LastIndex(svcAndMethod, "/") if pos < 0 { diff --git a/protodesc/protodesc_test.go b/protodesc/protodesc_test.go index 16d95488..fe8d5311 100644 --- a/protodesc/protodesc_test.go +++ b/protodesc/protodesc_test.go @@ -6,39 +6,83 @@ import ( "github.com/stretchr/testify/assert" ) -func TestProtodesc_GetMethodDesc(t *testing.T) { +func TestProtodesc_GetMethodDescFromProto(t *testing.T) { t.Run("invalid path", func(t *testing.T) { - md, err := GetMethodDesc("pkg.Call", "invalid.proto", []string{}) + md, err := GetMethodDescFromProto("pkg.Call", "invalid.proto", []string{}) assert.Error(t, err) assert.Nil(t, md) }) t.Run("invalid call symbol", func(t *testing.T) { - md, err := GetMethodDesc("pkg.Call", "../testdata/greeter.proto", []string{}) + md, err := GetMethodDescFromProto("pkg.Call", "../testdata/greeter.proto", []string{}) assert.Error(t, err) assert.Nil(t, md) }) t.Run("invalid package", func(t *testing.T) { - md, err := GetMethodDesc("helloworld.pkg.SayHello", "../testdata/greeter.proto", []string{}) + md, err := GetMethodDescFromProto("helloworld.pkg.SayHello", "../testdata/greeter.proto", []string{}) assert.Error(t, err) assert.Nil(t, md) }) t.Run("invalid method", func(t *testing.T) { - md, err := GetMethodDesc("helloworld.Greeter.Foo", "../testdata/greeter.proto", []string{}) + md, err := GetMethodDescFromProto("helloworld.Greeter.Foo", "../testdata/greeter.proto", []string{}) assert.Error(t, err) assert.Nil(t, md) }) t.Run("valid symbol", func(t *testing.T) { - md, err := GetMethodDesc("helloworld.Greeter.SayHello", "../testdata/greeter.proto", []string{}) + md, err := GetMethodDescFromProto("helloworld.Greeter.SayHello", "../testdata/greeter.proto", []string{}) assert.NoError(t, err) assert.NotNil(t, md) }) t.Run("valid symbol slashes", func(t *testing.T) { - md, err := GetMethodDesc("helloworld.Greeter/SayHello", "../testdata/greeter.proto", []string{}) + md, err := GetMethodDescFromProto("helloworld.Greeter/SayHello", "../testdata/greeter.proto", []string{}) + assert.NoError(t, err) + assert.NotNil(t, md) + }) +} + +func TestProtodesc_GetMethodDescFromProtoSet(t *testing.T) { + t.Run("invalid path", func(t *testing.T) { + md, err := GetMethodDescFromProtoSet("pkg.Call", "invalid.protoset") + assert.Error(t, err) + assert.Nil(t, md) + }) + + t.Run("invalid call symbol", func(t *testing.T) { + md, err := GetMethodDescFromProtoSet("pkg.Call", "../testdata/bundle.protoset") + assert.Error(t, err) + assert.Nil(t, md) + }) + + t.Run("invalid package", func(t *testing.T) { + md, err := GetMethodDescFromProtoSet("helloworld.pkg.SayHello", "../testdata/bundle.protoset") + assert.Error(t, err) + assert.Nil(t, md) + }) + + t.Run("invalid method", func(t *testing.T) { + md, err := GetMethodDescFromProtoSet("helloworld.Greeter.Foo", "../testdata/bundle.protoset") + assert.Error(t, err) + assert.Nil(t, md) + }) + + t.Run("valid symbol", func(t *testing.T) { + md, err := GetMethodDescFromProtoSet("helloworld.Greeter.SayHello", "../testdata/bundle.protoset") + assert.NoError(t, err) + assert.NotNil(t, md) + }) + + t.Run("valid symbol proto 2", func(t *testing.T) { + md, err := GetMethodDescFromProtoSet("cap.Capper.Cap", "../testdata/bundle.protoset") + assert.NoError(t, err) + assert.NotNil(t, md) + }) + + t.Run("valid symbol slashes", func(t *testing.T) { + md, err := GetMethodDescFromProtoSet("helloworld.Greeter/SayHello", "../testdata/bundle.protoset") assert.NoError(t, err) assert.NotNil(t, md) }) diff --git a/requester_test.go b/requester_test.go index c4a69e9d..84ff9a2d 100644 --- a/requester_test.go +++ b/requester_test.go @@ -55,7 +55,7 @@ func TestRequesterUnary(t *testing.T) { defer s.Stop() - md, err := protodesc.GetMethodDesc("helloworld.Greeter.SayHello", "./testdata/greeter.proto", []string{}) + md, err := protodesc.GetMethodDescFromProto("helloworld.Greeter.SayHello", "./testdata/greeter.proto", []string{}) data := make(map[string]interface{}) data["name"] = "bob" @@ -151,7 +151,7 @@ func TestRequesterServerStreaming(t *testing.T) { defer s.Stop() - md, err := protodesc.GetMethodDesc("helloworld.Greeter.SayHellos", "./testdata/greeter.proto", []string{}) + md, err := protodesc.GetMethodDescFromProto("helloworld.Greeter.SayHellos", "./testdata/greeter.proto", []string{}) data := make(map[string]interface{}) data["name"] = "bob" @@ -189,7 +189,7 @@ func TestRequesterClientStreaming(t *testing.T) { defer s.Stop() - md, err := protodesc.GetMethodDesc("helloworld.Greeter.SayHelloCS", "./testdata/greeter.proto", []string{}) + md, err := protodesc.GetMethodDescFromProto("helloworld.Greeter.SayHelloCS", "./testdata/greeter.proto", []string{}) m1 := make(map[string]interface{}) m1["name"] = "bob" @@ -237,7 +237,7 @@ func TestRequesterBidi(t *testing.T) { defer s.Stop() - md, err := protodesc.GetMethodDesc("helloworld.Greeter.SayHelloBidi", "./testdata/greeter.proto", []string{}) + md, err := protodesc.GetMethodDescFromProto("helloworld.Greeter.SayHelloBidi", "./testdata/greeter.proto", []string{}) m1 := make(map[string]interface{}) m1["name"] = "bob" @@ -285,7 +285,7 @@ func TestRequesterUnarySecure(t *testing.T) { defer s.Stop() - md, err := protodesc.GetMethodDesc("helloworld.Greeter.SayHello", "./testdata/greeter.proto", []string{}) + md, err := protodesc.GetMethodDescFromProto("helloworld.Greeter.SayHello", "./testdata/greeter.proto", []string{}) data := make(map[string]interface{}) data["name"] = "bob" diff --git a/testdata/bundle.protoset b/testdata/bundle.protoset new file mode 100644 index 0000000000000000000000000000000000000000..a5cae19cc7b584a05fdadb7eb9512181e56a6dd5 GIT binary patch literal 512 zcmb7=y^6v>6onm?Xb#!tVj)(NLQ;vku;2<7TCBA`!5A<8kT|oG$l^QsfR@IXS=27w zxx@M9yE8bz3soGeV_zGUDsmo^@55Vg6saZuo;XQ~P(R9;+oCH1cbR9u2NWG0ovDpR z%t<1*wq(b(lVAOb5N;@SA*yAu^Wq*&;LN4gY-osDmZ;S)c6(l>LvN><+^_sgE>(PN z4Rgw-FkyFp$e%LN=1q7NX#707^o8b1o^w$dFMR`+o3F}{fuJ#5%O+*efuJcoi_Z3& MeHl6s17W}G1~ZPN5dZ)H literal 0 HcmV?d00001 diff --git a/testdata/bundle/cap.proto b/testdata/bundle/cap.proto new file mode 100644 index 00000000..eed68ef3 --- /dev/null +++ b/testdata/bundle/cap.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package cap; + +import "common.proto"; + +service Capper { + rpc Cap (common.HelloRequest) returns (common.HelloReply) {} +} \ No newline at end of file diff --git a/testdata/bundle/common.proto b/testdata/bundle/common.proto new file mode 100644 index 00000000..b98a6c94 --- /dev/null +++ b/testdata/bundle/common.proto @@ -0,0 +1,14 @@ +// common.proto +syntax = "proto3"; + +package common; + +// The request message containing the user's name. +message HelloRequest { + string name = 1; +} + +// The response message containing the greetings +message HelloReply { + string message = 1; +} \ No newline at end of file diff --git a/testdata/bundle/greeter.proto b/testdata/bundle/greeter.proto new file mode 100644 index 00000000..a51c5e80 --- /dev/null +++ b/testdata/bundle/greeter.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package helloworld; + +import "common.proto"; + +service Greeter { + rpc SayHello (common.HelloRequest) returns (common.HelloReply) {} + rpc SayHelloCS (stream common.HelloRequest) returns (common.HelloReply) {} + rpc SayHellos (common.HelloRequest) returns (stream common.HelloReply) {} + rpc SayHelloBidi (stream common.HelloRequest) returns (stream common.HelloReply) {} +} \ No newline at end of file