feat: support running gRPC test cases

cmd/run.go

@@ -252,9 +252,9 @@ func (o *runOption) runSuite(loader testing.Loader, dataContext map[string]inter
 			ctxWithTimeout, _ := context.WithTimeout(ctx, o.requestTimeout)
 			ctxWithTimeout = context.WithValue(ctxWithTimeout, runner.ContextKey("").ParentDir(), loader.GetContext())
 
-			simpleRunner := runner.NewSimpleTestCaseRunner()
-			simpleRunner.WithTestReporter(o.reporter)
-			if output, err = simpleRunner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError {
+			runner := runner.GetTestSuiteRunner(testSuite)
+			runner.WithTestReporter(o.reporter)
+			if output, err = runner.RunTestCase(&testCase, dataContext, ctxWithTimeout); err != nil && !o.requestIgnoreError {
 				err = fmt.Errorf("failed to run '%s', %v", testCase.Name, err)
 				return
 			} else {

docs/api-testing-schema.json

@@ -45,13 +45,38 @@
             "properties": {
                 "kind": {
                     "type": "string",
-                    "enum": ["openapi", "swagger"]
+                    "enum": [
+                        "openapi",
+                        "swagger"
+                    ]
                 },
                 "url": {
                     "type": "string",
                     "qt-uri-protocols": [
-                        "https", "http"
+                        "https",
+                        "http"
                     ]
+                },
+                "grpc": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "properties": {
+                        "import": {
+                            "type": "array",
+                            "items": {
+                                "type": "string"
+                            }
+                        },
+                        "protofile": {
+                            "type": "string"
+                        },
+                        "protoset": {
+                            "type": "string"
+                        },
+                        "server_reflection": {
+                            "type": "boolean"
+                        }
+                    }
                 }
             }
         },
@@ -123,10 +148,16 @@
                     "qt-uri-protocols": [
                         "https"
                     ]
-                }, 
+                },
                 "method": {
                     "type": "string",
-                    "enum": ["GET", "POST", "PUT", "PATCH", "DELETE"]
+                    "enum": [
+                        "GET",
+                        "POST",
+                        "PUT",
+                        "PATCH",
+                        "DELETE"
+                    ]
                 },
                 "query": {
                     "description": "HTTP request query",
@@ -175,4 +206,4 @@
             "title": "Job"
         }
     }
-}
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3
 	github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883
 	github.com/antonmedv/expr v1.12.1
+	github.com/bufbuild/protocompile v0.6.0
 	github.com/cucumber/godog v0.12.6
 	github.com/ghodss/yaml v1.0.0
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0
@@ -14,12 +15,12 @@ require (
 	github.com/linuxsuren/go-fake-runtime v0.0.1
 	github.com/linuxsuren/unstructured v0.0.1
 	github.com/spf13/cobra v1.6.1
-	github.com/stretchr/testify v1.8.2
+	github.com/stretchr/testify v1.8.4
 	github.com/tidwall/gjson v1.14.4
 	github.com/xeipuuv/gojsonschema v1.2.0
-	golang.org/x/sync v0.1.0
+	golang.org/x/sync v0.3.0
 	google.golang.org/grpc v1.55.0
-	google.golang.org/protobuf v1.30.0
+	google.golang.org/protobuf v1.31.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -8,6 +8,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antonmedv/expr v1.12.1 h1:GTGrGN1kxxb+le0uQKaFRK8By4cvq1sleUCGE/U6hHg=
 github.com/antonmedv/expr v1.12.1/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
+github.com/bufbuild/protocompile v0.6.0 h1:Uu7WiSQ6Yj9DbkdnOe7U4mNKp58y9WDMKDn28/ZlunY=
+github.com/bufbuild/protocompile v0.6.0/go.mod h1:YNP35qEYoYGme7QMtz5SBCoN4kL4g12jTtjuzRNdjpE=
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -106,7 +108,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -115,8 +116,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
-github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
 github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
@@ -145,8 +146,8 @@ golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -179,8 +180,8 @@ google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag=
 google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
-google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

go.work.sum

@@ -746,6 +746,8 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/linuxsuren/go-fake-runtime v0.0.0-20230426144714-1a7a0d160d3f h1:TfAzkLxq/agwMBbccTx/f/dlmFWIBLWRGCWjI4IOlK8=
+github.com/linuxsuren/go-fake-runtime v0.0.0-20230426144714-1a7a0d160d3f/go.mod h1:zmh6J78hSnWZo68faMA2eKOdaEp8eFbERHi3ZB9xHCQ=
 github.com/lyft/protoc-gen-star v0.6.0/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
 github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
 github.com/lyft/protoc-gen-star/v2 v2.0.1/go.mod h1:RcCdONR2ScXaYnQC5tUzxzlpA3WVYF7/opLeUgcQs/o=
@@ -936,6 +938,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/compare/compare.go

@@ -0,0 +1,66 @@
+// Package compare provides basic functions to compare JSON object and array.
+// Currently we use gjson to store the JSON data. Please check tidwall/gjson.
+package compare
+
+import (
+	"fmt"
+	"reflect"
+	"strconv"
+
+	"github.com/tidwall/gjson"
+)
+
+// Object compares two JSON object.
+func Object(field string, expect, actul map[string]gjson.Result) error {
+	var errs error
+	for k, ev := range expect {
+		av, ok := actul[k]
+		if !ok {
+			errs = JoinErr(errs, newNoEqualErr(field, fmt.Errorf("field %s is not exist", k)))
+			continue
+		}
+
+		err := Element(k, ev, av)
+		if err != nil {
+			errs = JoinErr(errs, newNoEqualErr(field, err))
+		}
+	}
+	return errs
+}
+
+// Array compares two JSON Array.
+func Array(field string, expect, actul []gjson.Result) error {
+	var errs error
+	if l1, l2 := len(expect), len(actul); l1 != l2 {
+		return newNoEqualErr(field, fmt.Errorf("length is not equal, expect %v fields but got %v", l1, l2))
+	}
+
+	for i := range expect {
+		err := Element(strconv.Itoa(i), expect[i], actul[i])
+		if err != nil {
+			errs = JoinErr(errs, newNoEqualErr(field, err))
+		}
+	}
+	return errs
+}
+
+// Element compares two JSON Element.
+func Element(field string, expect, actul gjson.Result) error {
+	if expect.Type != actul.Type {
+		return newNoEqualErr(field, fmt.Errorf("expect type %s but got %v", expect.Type.String(), actul.Type.String()))
+	}
+
+	if expect.IsObject() {
+		return Object(field, expect.Map(), actul.Map())
+	}
+
+	if expect.IsArray() {
+		return Array(field, expect.Array(), actul.Array())
+	}
+
+	if !reflect.DeepEqual(expect.Value(), actul.Value()) {
+		return newNoEqualErr(field, fmt.Errorf("expect %v but got %v", expect.Value(), actul.Value()))
+	}
+
+	return nil
+}

pkg/compare/compare_test.go

@@ -0,0 +1,38 @@
+package compare
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/tidwall/gjson"
+)
+
+func TestElement(t *testing.T) {
+	exp := `{
+		"data": [
+		  {
+			"key": "hell",
+			"value": "func() strin"
+		  }
+		]
+	  }
+	`
+	act := `
+	  {
+		"data": [
+		  {
+			"key": "hello",
+			"value": "func() string"
+		  }
+		]
+	  }`
+	expect := gjson.Parse(exp)
+	actul := gjson.Parse(act)
+
+	err := Element("TestElement", expect, actul)
+
+	expmsg1 := "compare: field TestElement.data.0.value: expect func() strin but got func() string"
+	expmsg2 := "compare: field TestElement.data.0.key: expect hell but got hello"
+	assert.Contains(t, err.Error(), expmsg1)
+	assert.Contains(t, err.Error(), expmsg2)
+}

pkg/compare/error.go

@@ -0,0 +1,120 @@
+package compare
+
+import (
+	"fmt"
+	"strings"
+)
+
+type noEqualErr struct {
+	Path    []string
+	Message string
+	NeqErrs *noEqualErrs
+}
+
+type noEqualErrs struct {
+	errs []error
+}
+
+type task struct {
+	path []string
+	errs []error
+}
+
+func (e *noEqualErr) Error() string {
+	if e.NeqErrs == nil {
+		return fmt.Sprintf("compare: field %s: %s", strings.Join(e.Path, "."), e.Message)
+	}
+
+	msg := []string{}
+	q := []*task{{
+		path: e.Path,
+		errs: e.NeqErrs.errs,
+	}}
+	iter := 0
+	end := len(e.NeqErrs.errs)
+	for {
+		if iter == end {
+			iter = 0
+
+			q[0] = nil
+			q = q[1:]
+			if len(q) == 0 {
+				break
+			}
+
+			end = len(q[0].errs)
+			continue
+		}
+
+		v := q[0].errs[iter]
+		switch v.(type) {
+		case *noEqualErr:
+			if v.(*noEqualErr).NeqErrs != nil {
+				q = append(q, &task{append(q[0].path, v.(*noEqualErr).Path...), v.(*noEqualErr).NeqErrs.errs})
+			} else {
+				msg = append(msg,
+					fmt.Sprintf("compare: field %s: %s",
+						strings.Join(append(q[0].path, v.(*noEqualErr).Path...), "."), v.(*noEqualErr).Message))
+			}
+		case *noEqualErrs:
+			q = append(q, &task{path: q[0].path, errs: v.(*noEqualErrs).errs})
+		}
+		iter++
+	}
+	return strings.Join(msg, "\n")
+}
+
+// newNoEqualErr returns a NoEqualErr.
+func newNoEqualErr(field string, err error) error {
+	noEqerr := &noEqualErr{
+		Path:    []string{field},
+		Message: "",
+	}
+
+	if err != nil {
+		if v, ok := err.(*noEqualErr); ok {
+			noEqerr.Path = append(noEqerr.Path, v.Path...)
+			if v.Message != "" {
+				noEqerr.Message = v.Message
+			}
+		} else if v, ok := err.(*noEqualErrs); ok {
+			noEqerr.NeqErrs = v
+		} else {
+			noEqerr.Message = err.Error()
+		}
+	}
+	return noEqerr
+}
+
+// JoinErr returns an error that warp the given errors.
+func JoinErr(errs ...error) error {
+	n := 0
+	for _, err := range errs {
+		if err != nil {
+			n++
+		}
+	}
+	if n == 0 {
+		return nil
+	}
+	e := &noEqualErrs{
+		errs: make([]error, 0, n),
+	}
+	for _, err := range errs {
+		if err != nil {
+			e.errs = append(e.errs, err)
+		}
+	}
+	return e
+}
+
+func (e *noEqualErrs) Error() string {
+	var b []byte
+	for i, err := range e.errs {
+		if i > 0 {
+			b = append(b, '\n')
+		}
+		b = append(b, err.Error()...)
+	}
+	return string(b)
+}